From 7004c218e78fa854341f84789d6489a761202068 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 10:53:33 +0530 Subject: [PATCH 001/311] Sanketika-obsrv/issue-tracker: #OBS-150: added connector-registry route, controller,and required configs --- api-service/src/app.ts | 1 + api-service/src/v2/configs/Config.ts | 18 ++- .../ConnectorRegistryStreamController.ts | 113 ++++++++++++++++++ .../GenerateSignedURL/GenerateSignedURL.ts | 12 +- api-service/src/v2/routes/Router.ts | 2 + 5 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts diff --git a/api-service/src/app.ts b/api-service/src/app.ts index bc7aad96..177d106f 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -24,6 +24,7 @@ app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.set("queryServices", services); +app.use(bodyParser.raw({ type: "application/octet-stream", limit: "500mb" })) loadExtensions(app) .finally(() => { diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 094b1924..c96665d0 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -69,11 +69,12 @@ export const config = { } }, "cloud_config": { - "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure - "cloud_storage_region": process.env.cloud_storage_region || "", // Region for the cloud provider storage - "cloud_storage_config": process.env.cloud_storage_config ? JSON.parse(process.env.cloud_storage_config) : {}, // Respective credentials object for cloud provider. Optional if service account provided - "container": process.env.container || "container", // Storage container/bucket name - "container_prefix": process.env.container_prefix || "", // Path to the folder inside container/bucket. Empty if data at root level + "cloud_storage_provider": process.env.cloud_storage_provider || "gcloud", // Supported providers - AWS, GCP, Azure + "cloud_storage_region": process.env.cloud_storage_region || "us-east-2", // Region for the cloud provider storage + "cloud_storage_config": process.env.cloud_storage_config || {}, // Respective credentials object for cloud provider. Optional if service account provided + "container": process.env.container || "exhaust-test-bucket", // Storage container/bucket name + "connector_container": process.env.container || "connector-registry", + "container_prefix": process.env.container_prefix || "connector-registry", // Path to the folder inside container/bucket. Empty if data at root level "storage_url_expiry": process.env.storage_url_expiry ? parseInt(process.env.storage_url_expiry) : 3600, // in seconds, Default 1hr of expiry for Signed URLs. "maxQueryDateRange": process.env.exhaust_query_range ? parseInt(process.env.exhaust_query_range) : 31, // in days. Defines the maximum no. of days the files can be fetched "exclude_exhaust_types": process.env.exclude_exhaust_types ? process.env.exclude_exhaust_types.split(",") : ["system-stats", "masterdata-system-stats", "system-events",] // list of folder type names to skip exhaust service @@ -90,7 +91,12 @@ export const config = { }, "command_service_config": { "host": process.env.command_service_host || "http://localhost", - "port": parseInt(process.env.command_service_port || "8000"), + "port": parseInt(process.env.command_service_port || "9999"), "path": process.env.command_service_path || "/system/v1/dataset/command" + }, + "obsrv_api_service_config": { + "host": process.env.obser_api_service_host || "http://localhost", + "port": parseInt(process.env.obser_api_service_port || "3007"), + "generate_url_path": process.env.generate_url_path || "/v2/files/generate-url" } } diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts new file mode 100644 index 00000000..702d79a9 --- /dev/null +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -0,0 +1,113 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import _ from "lodash"; +import logger from "../../logger"; +import { config } from "../../configs/Config"; +import axios from "axios"; +import httpStatus from "http-status"; +import busboy from "busboy"; +import { v4 } from "uuid" +import { PassThrough } from "stream"; +import { URLAccess } from "../../types/SampleURLModel"; + +export const apiId = "api.files.generate-url"; +export const code = "FILES_GENERATE_URL_FAILURE"; + +const apiServiceHost = _.get(config, ["obsrv_api_service_config", "host"]); +const apiServicePort = _.get(config, ["obsrv_api_service_config", "port"]); +const generateSignedURLPath = _.get(config, ["obsrv_api_service_config", "generate_url_path"]); + +const getGenerateSignedURLRequestBody = (files: string[], access: string) => ({ + id: apiId, + ver: "v2", + ts: new Date().toISOString(), + params: { + msgid: v4() + }, + request: { + files, + access: access || URLAccess.Read, + type: "connector" + } +}); + +const connectorRegistryStream = async (req: Request, res: Response) => { + const resmsgid = _.get(res, "resmsgid"); + try { + const uploadStreamResponse: any = await uploadStream(req); + // console.log({ uploadStreamResponse }) + // const readPreSignedUrlsPromises = uploadStreamResponse.map(async (filePath: any) => { + // const readPreSignedUrl: any = await generatePresignedUrl(filePath, URLAccess.Read); + // return readPreSignedUrl[0]?.preSignedUrl; + // }); + // const preSignedReadUrls = await Promise.all(readPreSignedUrlsPromises); + logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: "Successfully uploaded", preSignedReadUrls: uploadStreamResponse } }) + } catch (error: any) { + logger.error(error, apiId, resmsgid, code); + const statusCode = _.get(error, "statusCode", 500); + const errorMessage = statusCode === 500 ? { code, message: "Failed to upload" } : error; + ResponseHandler.errorResponse(errorMessage, req, res); + } +}; + +export const generatePresignedUrl = async (fileName: string, access: string) => { + try { + const requestBody = getGenerateSignedURLRequestBody([fileName], access); + const response = await axios.post(`${apiServiceHost}:${apiServicePort}${generateSignedURLPath}`, requestBody); + return response?.data?.result; + } + catch (err) { + throw err + } +}; + +const uploadStream = async (req: Request) => { + return new Promise((resolve, reject) => { + const filePromises: Promise[] = []; + const bb = busboy({ headers: req.headers }); + console.log(bb, true); + const match: any[] = []; + + bb.on("file", async (name: any, file: any, info: any) => { + const processFile = async () => { + const fileName = info?.filename; + const preSignedUrl: any = await generatePresignedUrl(fileName, URLAccess.Write); + const filePath = preSignedUrl[0]?.filePath + const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; + match.push(...filePath.match(regex)); + const pass = new PassThrough(); + file.pipe(pass); + const fileBuffer = await streamToBuffer(pass); + await axios.put(preSignedUrl[0]?.preSignedUrl, fileBuffer, { + headers: { + "Content-Type": info.mimeType, + "Content-Length": fileBuffer.length, + } + }); + }; + filePromises.push(processFile()); + }); + bb.on("close", async () => { + try { + await Promise.all(filePromises); + resolve(match); + } catch (error) { + reject(error); + } + }); + bb.on("error", reject); + req.pipe(bb); + }) +} + +const streamToBuffer = (stream: PassThrough): Promise => { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("error", reject); + }); +}; + +export default connectorRegistryStream; diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts index 6182f9b3..7979cafa 100644 --- a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -15,11 +15,13 @@ import path from "path"; export const apiId = "api.files.generate-url" export const code = "FILES_GENERATE_URL_FAILURE" const maxFiles = config.presigned_url_configs.maxFiles +let containerType: string; const generateSignedURL = async (req: Request, res: Response) => { const requestBody = req.body const msgid = _.get(req, ["body", "params", "msgid"]); const resmsgid = _.get(res, "resmsgid"); + containerType = _.get(req, ["body", "request", "type"]); try { const isRequestValid: Record = schemaValidation(req.body, GenerateURL) if (!isRequestValid.isValid) { @@ -75,7 +77,15 @@ const generateSignedURL = async (req: Request, res: Response) => { } const getFilePath = (file: string) => { - return `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}` + const datasetUploadPath = `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}`; + const connectorUploadPath = `${config.cloud_config.connector_container}/${file}`; + + const paths: Record = { + "dataset": datasetUploadPath, + "connector": connectorUploadPath + }; + + return paths[containerType] || datasetUploadPath; } const transformFileNames = (fileList: Array, access: string): Record => { diff --git a/api-service/src/v2/routes/Router.ts b/api-service/src/v2/routes/Router.ts index ebdbe423..7ad01a2a 100644 --- a/api-service/src/v2/routes/Router.ts +++ b/api-service/src/v2/routes/Router.ts @@ -20,6 +20,7 @@ import { eventValidation } from "../controllers/EventValidation/EventValidation" import GenerateSignedURL from "../controllers/GenerateSignedURL/GenerateSignedURL"; import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; import DatasetStatusTansition from "../controllers/DatasetStatusTransition/DatasetStatusTransition"; +import connectorRegistryStream from "../controllers/ConnectorRegistryStream/ConnectorRegistryStreamController"; export const router = express.Router(); @@ -39,6 +40,7 @@ router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); +router.post("/connector/stream/upload", setDataToRequestObject("api.connector.stream.upload"), onRequest({ entity: Entity.Management }), connectorRegistryStream); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); From ab2a37a492d076fb8b476e2a322bf1a9aefee19a Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 11:18:45 +0530 Subject: [PATCH 002/311] Sanketika-obsrv/issue-tracker: #OBS-150: added packages --- api-service/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api-service/package.json b/api-service/package.json index e4754097..1df0f26e 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -28,6 +28,7 @@ "aws-sdk": "^2.1348.0", "axios": "^1.6.0", "body-parser": "^1.20.2", + "busboy": "^1.6.0", "compression": "^1.7.4", "dateformat": "2.0.0", "express": "^4.18.2", @@ -43,6 +44,7 @@ "moment": "^2.29.4", "multiparty": "4.2.1", "node-sql-parser": "^5.1.0", + "path": "^0.12.7", "pg": "^8.11.3", "pg-hstore": "^2.3.4", "prom-client": "^14.2.0", @@ -58,6 +60,7 @@ "@babel/traverse": "7.23.2" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/chai": "^4.3.3", "@types/chai-as-promised": "^7.1.5", "@types/chai-spies": "^1.0.3", From 6edf484465f0623c1a7366c7df5d8f90f9281b01 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 11:31:49 +0530 Subject: [PATCH 003/311] Sanketika-obsrv/issue-tracker: #OBS-150: added download link as response --- .../ConnectorRegistryStreamController.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 702d79a9..3f081b3d 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -36,13 +36,13 @@ const connectorRegistryStream = async (req: Request, res: Response) => { try { const uploadStreamResponse: any = await uploadStream(req); // console.log({ uploadStreamResponse }) - // const readPreSignedUrlsPromises = uploadStreamResponse.map(async (filePath: any) => { - // const readPreSignedUrl: any = await generatePresignedUrl(filePath, URLAccess.Read); - // return readPreSignedUrl[0]?.preSignedUrl; - // }); - // const preSignedReadUrls = await Promise.all(readPreSignedUrlsPromises); + const readPreSignedUrlsPromises = uploadStreamResponse.map(async (filePath: any) => { + const readPreSignedUrl: any = await generatePresignedUrl(filePath, URLAccess.Read); + return readPreSignedUrl[0]?.preSignedUrl; + }); + const preSignedReadUrls = await Promise.all(readPreSignedUrlsPromises); logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: "Successfully uploaded", preSignedReadUrls: uploadStreamResponse } }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: "Successfully uploaded", preSignedReadUrls: preSignedReadUrls } }) } catch (error: any) { logger.error(error, apiId, resmsgid, code); const statusCode = _.get(error, "statusCode", 500); From 23bd5b78e742d09e0c13ab2f2eaf116df1a83c6f Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 11:32:26 +0530 Subject: [PATCH 004/311] Sanketika-obsrv/issue-tracker: #OBS-150: deciding path based on type --- .../v2/controllers/GenerateSignedURL/GenerateSignedURL.ts | 2 +- .../GenerateSignedURLValidationSchema.json | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts index 7979cafa..04f60329 100644 --- a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -78,7 +78,7 @@ const generateSignedURL = async (req: Request, res: Response) => { const getFilePath = (file: string) => { const datasetUploadPath = `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}`; - const connectorUploadPath = `${config.cloud_config.connector_container}/${file}`; + const connectorUploadPath = `${config.cloud_config.container}/${config.cloud_config.container_prefix}/${file}`; const paths: Record = { "dataset": datasetUploadPath, diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json index 553a784e..77db2e33 100644 --- a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json +++ b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json @@ -42,6 +42,13 @@ "read", "write" ] + }, + "type": { + "type": "string", + "enum": [ + "dataset", + "connector" + ] } }, "additionalProperties": false, From f324134c41e21e89fb59fe9d5db594ed67b1dde0 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 11:33:13 +0530 Subject: [PATCH 005/311] Sanketika-obsrv/issue-tracker: #OBS-150: removed container config --- api-service/src/v2/configs/Config.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index c96665d0..2c0e7300 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -69,11 +69,10 @@ export const config = { } }, "cloud_config": { - "cloud_storage_provider": process.env.cloud_storage_provider || "gcloud", // Supported providers - AWS, GCP, Azure + "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure "cloud_storage_region": process.env.cloud_storage_region || "us-east-2", // Region for the cloud provider storage "cloud_storage_config": process.env.cloud_storage_config || {}, // Respective credentials object for cloud provider. Optional if service account provided "container": process.env.container || "exhaust-test-bucket", // Storage container/bucket name - "connector_container": process.env.container || "connector-registry", "container_prefix": process.env.container_prefix || "connector-registry", // Path to the folder inside container/bucket. Empty if data at root level "storage_url_expiry": process.env.storage_url_expiry ? parseInt(process.env.storage_url_expiry) : 3600, // in seconds, Default 1hr of expiry for Signed URLs. "maxQueryDateRange": process.env.exhaust_query_range ? parseInt(process.env.exhaust_query_range) : 31, // in days. Defines the maximum no. of days the files can be fetched From 6f743d30e778bf877d8bcbc74429be681c05ec7c Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 11:55:49 +0530 Subject: [PATCH 006/311] Sanketika-obsrv/issue-tracker: #OBS-150: refactored container saving path --- .../src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts index 04f60329..e2bc2855 100644 --- a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -77,8 +77,8 @@ const generateSignedURL = async (req: Request, res: Response) => { } const getFilePath = (file: string) => { - const datasetUploadPath = `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}`; - const connectorUploadPath = `${config.cloud_config.container}/${config.cloud_config.container_prefix}/${file}`; + const datasetUploadPath = `${config.presigned_url_configs.service}/user_uploads/${file}`; + const connectorUploadPath = `${config.cloud_config.container_prefix}/${file}`; const paths: Record = { "dataset": datasetUploadPath, From 90a5f88a7b846ef2f673a719f71311a4225e2286 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 16:47:04 +0530 Subject: [PATCH 007/311] Sanketika-obsrv/issue-tracker: #OBS-150: removed bodyparser --- api-service/src/app.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 177d106f..bc7aad96 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -24,7 +24,6 @@ app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.set("queryServices", services); -app.use(bodyParser.raw({ type: "application/octet-stream", limit: "500mb" })) loadExtensions(app) .finally(() => { From 795397ffdde68fe25e34be1114c7c752ed385b25 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 16:52:45 +0530 Subject: [PATCH 008/311] Sanketika-obsrv/issue-tracker: #OBS-150: implementation of calling command api with relative path as request body --- .../ConnectorRegistryStreamController.ts | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 3f081b3d..082d7a0e 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -9,6 +9,7 @@ import busboy from "busboy"; import { v4 } from "uuid" import { PassThrough } from "stream"; import { URLAccess } from "../../types/SampleURLModel"; +import { ErrorObject } from "../../types/ResponseModel"; export const apiId = "api.files.generate-url"; export const code = "FILES_GENERATE_URL_FAILURE"; @@ -17,6 +18,10 @@ const apiServiceHost = _.get(config, ["obsrv_api_service_config", "host"]); const apiServicePort = _.get(config, ["obsrv_api_service_config", "port"]); const generateSignedURLPath = _.get(config, ["obsrv_api_service_config", "generate_url_path"]); +const commandServiceHost = _.get(config, ["command_service_config", "host"]); +const commandServicePort = _.get(config, ["command_service_config", "port"]); +const registryUrl = _.get(config, ["command_service_config", "connector_registry_path"]) + const getGenerateSignedURLRequestBody = (files: string[], access: string) => ({ id: apiId, ver: "v2", @@ -35,18 +40,20 @@ const connectorRegistryStream = async (req: Request, res: Response) => { const resmsgid = _.get(res, "resmsgid"); try { const uploadStreamResponse: any = await uploadStream(req); - // console.log({ uploadStreamResponse }) - const readPreSignedUrlsPromises = uploadStreamResponse.map(async (filePath: any) => { - const readPreSignedUrl: any = await generatePresignedUrl(filePath, URLAccess.Read); - return readPreSignedUrl[0]?.preSignedUrl; - }); - const preSignedReadUrls = await Promise.all(readPreSignedUrlsPromises); + const registryRequestBody = { + relative_path: uploadStreamResponse[0] + } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: "Successfully uploaded", preSignedReadUrls: preSignedReadUrls } }) + const registryResponse = await axios.post(`${commandServiceHost}:${commandServicePort}${registryUrl}`, registryRequestBody); + logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { logger.error(error, apiId, resmsgid, code); - const statusCode = _.get(error, "statusCode", 500); - const errorMessage = statusCode === 500 ? { code, message: "Failed to upload" } : error; + let errorMessage = error; + const statusCode = _.get(error, "statusCode") + if (!statusCode || statusCode == 500) { + errorMessage = { code, message: "Failed to read dataset" } + } ResponseHandler.errorResponse(errorMessage, req, res); } }; @@ -58,7 +65,12 @@ export const generatePresignedUrl = async (fileName: string, access: string) => return response?.data?.result; } catch (err) { - throw err + throw { + code: "FILES_GENERATE_URL_FAILURE", + message: "Failed to generate sample urls", + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject } }; @@ -66,10 +78,21 @@ const uploadStream = async (req: Request) => { return new Promise((resolve, reject) => { const filePromises: Promise[] = []; const bb = busboy({ headers: req.headers }); - console.log(bb, true); const match: any[] = []; + let fileCount = 0; bb.on("file", async (name: any, file: any, info: any) => { + if (fileCount > 0) { + // If more than one file is detected, reject the request + bb.emit("error", reject({ + code: "FAILED_TO_UPLOAD", + message: "Uploading multiple files are not allowed", + statusCode: 400, + errCode: "BAD_REQUEST" + })); + return + } + fileCount++; const processFile = async () => { const fileName = info?.filename; const preSignedUrl: any = await generatePresignedUrl(fileName, URLAccess.Write); From 62261dd2885d61e9cae97d1f3787ac890cc50eb5 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 17:47:28 +0530 Subject: [PATCH 009/311] Sanketika-obsrv/issue-tracker: #OBS-150: updated error codes and added error handling --- .../ConnectorRegistryStreamController.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 082d7a0e..20c7d3c2 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -12,7 +12,7 @@ import { URLAccess } from "../../types/SampleURLModel"; import { ErrorObject } from "../../types/ResponseModel"; export const apiId = "api.files.generate-url"; -export const code = "FILES_GENERATE_URL_FAILURE"; +export const code = "FAILED_TO_REGISTER_CONNECTOR"; const apiServiceHost = _.get(config, ["obsrv_api_service_config", "host"]); const apiServicePort = _.get(config, ["obsrv_api_service_config", "port"]); @@ -48,11 +48,12 @@ const connectorRegistryStream = async (req: Request, res: Response) => { logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { + const errMessage = _.get(error, "response.data.error.message") logger.error(error, apiId, resmsgid, code); let errorMessage = error; const statusCode = _.get(error, "statusCode") if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to read dataset" } + errorMessage = { code, message: errMessage || "Failed to register connector" } } ResponseHandler.errorResponse(errorMessage, req, res); } @@ -78,7 +79,7 @@ const uploadStream = async (req: Request) => { return new Promise((resolve, reject) => { const filePromises: Promise[] = []; const bb = busboy({ headers: req.headers }); - const match: any[] = []; + const relative_path: any[] = []; let fileCount = 0; bb.on("file", async (name: any, file: any, info: any) => { @@ -97,8 +98,8 @@ const uploadStream = async (req: Request) => { const fileName = info?.filename; const preSignedUrl: any = await generatePresignedUrl(fileName, URLAccess.Write); const filePath = preSignedUrl[0]?.filePath - const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; - match.push(...filePath.match(regex)); + const fileNameExtracted = extractFileNameFromPath(filePath); + relative_path.push(...fileNameExtracted); const pass = new PassThrough(); file.pipe(pass); const fileBuffer = await streamToBuffer(pass); @@ -114,12 +115,16 @@ const uploadStream = async (req: Request) => { bb.on("close", async () => { try { await Promise.all(filePromises); - resolve(match); + resolve(relative_path); } catch (error) { - reject(error); + reject({ + code: "FAILED_TO_UPLOAD", + message: "Fail to upload a file", + statusCode: 400, + errCode: "BAD_REQUEST" + }); } }); - bb.on("error", reject); req.pipe(bb); }) } @@ -133,4 +138,9 @@ const streamToBuffer = (stream: PassThrough): Promise => { }); }; +const extractFileNameFromPath = (filePath: string): string[] => { + const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; + return filePath.match(regex) || []; +}; + export default connectorRegistryStream; From 38e2f1ead1318ea450cb5e33ea397829335c16e4 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 17:50:55 +0530 Subject: [PATCH 010/311] Sanketika-obsrv/issue-tracker: #OBS-150: added config route for command api --- api-service/src/v2/configs/Config.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 2c0e7300..2d59c935 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -91,11 +91,12 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "9999"), - "path": process.env.command_service_path || "/system/v1/dataset/command" + "path": process.env.command_service_path || "/system/v1/dataset/command", + "connector_registry_path": process.env.connector_registry_path || "/connector/v1/register", }, "obsrv_api_service_config": { "host": process.env.obser_api_service_host || "http://localhost", "port": parseInt(process.env.obser_api_service_port || "3007"), - "generate_url_path": process.env.generate_url_path || "/v2/files/generate-url" + "generate_url_path": process.env.generate_url_path || "/v2/files/generate-url", } } From acfd5de918340eb659014a3b9c81602f4bda88a6 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 18:05:20 +0530 Subject: [PATCH 011/311] Sanketika-obsrv/issue-tracker: #OBS-150: added promise reject on error --- .../ConnectorRegistryStreamController.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 20c7d3c2..fe2ee46b 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -79,7 +79,7 @@ const uploadStream = async (req: Request) => { return new Promise((resolve, reject) => { const filePromises: Promise[] = []; const bb = busboy({ headers: req.headers }); - const relative_path: any[] = []; + const match: any[] = []; let fileCount = 0; bb.on("file", async (name: any, file: any, info: any) => { @@ -98,8 +98,8 @@ const uploadStream = async (req: Request) => { const fileName = info?.filename; const preSignedUrl: any = await generatePresignedUrl(fileName, URLAccess.Write); const filePath = preSignedUrl[0]?.filePath - const fileNameExtracted = extractFileNameFromPath(filePath); - relative_path.push(...fileNameExtracted); + const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; + match.push(...filePath.match(regex)); const pass = new PassThrough(); file.pipe(pass); const fileBuffer = await streamToBuffer(pass); @@ -115,7 +115,7 @@ const uploadStream = async (req: Request) => { bb.on("close", async () => { try { await Promise.all(filePromises); - resolve(relative_path); + resolve(match); } catch (error) { reject({ code: "FAILED_TO_UPLOAD", @@ -125,6 +125,7 @@ const uploadStream = async (req: Request) => { }); } }); + bb.on("error", reject); req.pipe(bb); }) } @@ -138,9 +139,4 @@ const streamToBuffer = (stream: PassThrough): Promise => { }); }; -const extractFileNameFromPath = (filePath: string): string[] => { - const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; - return filePath.match(regex) || []; -}; - export default connectorRegistryStream; From 4808e68708a8cd4af74d546d1a1c697fc5db7d27 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 18:10:47 +0530 Subject: [PATCH 012/311] Sanketika-obsrv/issue-tracker: #OBS-150: added regex to find filename in presigned link --- .../ConnectorRegistryStreamController.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index fe2ee46b..941e9dfb 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -79,7 +79,7 @@ const uploadStream = async (req: Request) => { return new Promise((resolve, reject) => { const filePromises: Promise[] = []; const bb = busboy({ headers: req.headers }); - const match: any[] = []; + const relative_path: any[] = []; let fileCount = 0; bb.on("file", async (name: any, file: any, info: any) => { @@ -98,8 +98,8 @@ const uploadStream = async (req: Request) => { const fileName = info?.filename; const preSignedUrl: any = await generatePresignedUrl(fileName, URLAccess.Write); const filePath = preSignedUrl[0]?.filePath - const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; - match.push(...filePath.match(regex)); + const fileNameExtracted = extractFileNameFromPath(filePath); + relative_path.push(...fileNameExtracted); const pass = new PassThrough(); file.pipe(pass); const fileBuffer = await streamToBuffer(pass); @@ -115,7 +115,7 @@ const uploadStream = async (req: Request) => { bb.on("close", async () => { try { await Promise.all(filePromises); - resolve(match); + resolve(relative_path); } catch (error) { reject({ code: "FAILED_TO_UPLOAD", @@ -139,4 +139,9 @@ const streamToBuffer = (stream: PassThrough): Promise => { }); }; +const extractFileNameFromPath = (filePath: string): string[] => { + const regex = /(?<=\/)[^/]+\.[^/]+(?=\/|$)/g; + return filePath.match(regex) || []; +}; + export default connectorRegistryStream; From 0aaec483caf2b2540cce651c1cf46b700884d319 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 25 Jun 2024 18:21:25 +0530 Subject: [PATCH 013/311] Sanketika-obsrv/issue-tracker: #OBS-150: changed default port --- api-service/src/v2/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 2d59c935..ddec9c29 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -96,7 +96,7 @@ export const config = { }, "obsrv_api_service_config": { "host": process.env.obser_api_service_host || "http://localhost", - "port": parseInt(process.env.obser_api_service_port || "3007"), + "port": parseInt(process.env.obser_api_service_port || "3000"), "generate_url_path": process.env.generate_url_path || "/v2/files/generate-url", } } From d68b780e9c62ccd3aec42f97d3239514c5f60a8c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 25 Jun 2024 19:29:16 +0530 Subject: [PATCH 014/311] Sanketika-Obsrv/issue-tracker#OBS-151: feat: Generate ingestion spec before publish dataset --- .../DatasetCreateValidationSchema.json | 34 +- .../DatasetStatusTransition.ts | 21 +- .../DatasetUpdate/DatasetUpdate.ts | 4 +- .../DatasetUpdateValidationSchema.json | 29 +- .../src/v2/services/DatasourceService.ts | 33 +- .../src/v2/services/IngestionService.ts | 22 +- .../DatasetCreate/Fixtures.ts | 18 +- .../DatasetManagement/DatasetList/Fixtures.ts | 6 +- .../DatasetManagement/DatasetRead/Fixtures.ts | 6 +- .../DatasetLive.spec.ts | 10 +- .../DatasetStatusTransition/Fixtures.ts | 384 ++++++++++-------- .../DatasetUpdate/DatasetDenorm.spec.ts | 9 +- .../DatasetTransformation.spec.ts | 9 +- .../DatasetUpdate/DatasetUpdate.spec.ts | 3 +- .../DatasetUpdate/Fixtures.ts | 87 +++- 15 files changed, 407 insertions(+), 268 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index b2248e1d..a8715500 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -130,9 +130,12 @@ "denorm_out_field": { "type": "string", "minLength": 1 + }, + "redis_db": { + "type": "number" } }, - "required": ["denorm_key", "denorm_out_field"], + "required": ["denorm_key", "denorm_out_field", "redis_db"], "additionalProperties": false } } @@ -171,7 +174,29 @@ "minLength": 1 }, "transformation_function": { - "type": "object" + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + }, + "condition": { + "oneOf": [ + { + "type": "string" + }, + { + "const": null + } + ] + } + }, + "additionalProperties": false, + "required": ["type", "expr"] }, "mode": { "type": "string", @@ -188,8 +213,7 @@ "required": [ "field_key", "transformation_function", - "mode", - "metadata" + "mode" ] } }, @@ -197,7 +221,7 @@ "type": "array", "items": { "type": "string", - "minLength":1 + "minLength": 1 } } }, diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 317b7ae4..818c360d 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import _ from "lodash"; import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { getDataset, getDraftDataset, setReqDatasetId } from "../../services/DatasetService"; +import { generateDataSource, getDataset, getDraftDataset, setReqDatasetId } from "../../services/DatasetService"; import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import StatusTransitionSchema from "./RequestValidationSchema.json"; @@ -34,7 +34,7 @@ const allowedTransitions = { const statusTransitionCommands = { Delete: ["DELETE_DRAFT_DATASETS"], ReadyToPublish: ["VALIDATE_DATASET_CONFIGS"], - Live: ["PUBLISH_DATASET"], + Live: ["GENERATE_INGESTION_SPEC", "PUBLISH_DATASET"], Retire: ["CHECK_DATASET_IS_DENORM", "SET_DATASET_TO_RETIRE", "DELETE_SUPERVISORS", "RESTART_PIPELINE"] } @@ -121,11 +121,10 @@ const fetchDataset = async (configs: Record) => { const executeTransition = async (configs: Record) => { const { transitionCommands, dataset, transact } = configs - const transitionPromises = _.map(transitionCommands, async command => { - const commandWorkflow = _.get(commandExecutors, command) - return commandWorkflow({ dataset, transact }) - }) - await Promise.all(transitionPromises) + for (const command of transitionCommands) { + const commandWorkflow = _.get(commandExecutors, command); + await commandWorkflow({ dataset, transact }); + } } //VALIDATE_DATASET_CONFIGS @@ -158,6 +157,13 @@ const deleteDraftRecords = async (config: Record) => { await DatasetDraft.destroy({ where: { id: dataset_id }, transaction: transact }) } +//GENERATE_INGESTION_SPEC +const generateDatasource = async (config: Record) => { + const { dataset } = config; + const dataSource = await generateDataSource(dataset); + return DatasourceDraft.upsert(dataSource) +} + //PUBLISH_DATASET const publishDataset = async (configs: Record) => { const { dataset } = configs @@ -225,6 +231,7 @@ const restartPipeline = async (config: Record) => { const commandExecutors = { DELETE_DRAFT_DATASETS: deleteDataset, PUBLISH_DATASET: publishDataset, + GENERATE_INGESTION_SPEC: generateDatasource, CHECK_DATASET_IS_DENORM: checkDatasetDenorm, SET_DATASET_TO_RETIRE: setDatasetRetired, DELETE_SUPERVISORS: deleteSupervisors, diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index d8504199..e5e53a6d 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -150,7 +150,7 @@ const checkDatasetExists = async (dataset_id: string, version_key: string): Prom if (datasetExists) { const validVersionKey = _.get(datasetExists, "version_key") const apiVersion = _.get(datasetExists, "api_version") - if (validVersionKey !== version_key && apiVersion) { + if (validVersionKey !== version_key && apiVersion === "v2") { return { isDatasetExists: true, datasetStatus: datasetExists.status, invalidVersionKey: true, validVersionKey } } return { isDatasetExists: true, datasetStatus: datasetExists.status } @@ -344,7 +344,7 @@ const setDenormConfigs = (newDenormPayload: Record, datasetDenormCo if (_.isEmpty(denorm_fields)) { return { denorm_fields } } - + const getDenormPayload = (action: string) => { return _.compact(_.flatten(_.map(denorm_fields, payload => { if (payload.action == action) return payload.values diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 76f1bbb1..b20c0a51 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -218,7 +218,29 @@ "minLength": 1 }, "transformation_function": { - "type": "object" + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + }, + "condition": { + "oneOf": [ + { + "type": "string" + }, + { + "const": null + } + ] + } + }, + "additionalProperties": false, + "required": ["type", "expr"] }, "mode": { "type": "string", @@ -293,6 +315,9 @@ "denorm_out_field": { "type": "string", "minLength": 1 + }, + "redis_db": { + "type": "number" } }, "required": ["denorm_out_field"] @@ -314,7 +339,7 @@ "then": { "properties": { "values": { - "required": ["denorm_key"] + "required": ["denorm_key", "redis_db"] } } } diff --git a/api-service/src/v2/services/DatasourceService.ts b/api-service/src/v2/services/DatasourceService.ts index 96c8f294..05e1b6dc 100644 --- a/api-service/src/v2/services/DatasourceService.ts +++ b/api-service/src/v2/services/DatasourceService.ts @@ -63,36 +63,15 @@ export const getDatasource = async (datasetId: string) => { } export const getUpdatedSchema = async (configs: Record) => { - const { id, transformation_config, denorm_config, data_schema, action, indexCol = ingestionConfig.indexCol["Event Arrival Time"] } = configs - const existingTransformations = await DatasetTransformationsDraft.findAll({ where: { dataset_id: id }, raw: true }) - let resultantTransformations: any[] = [] - if (action === "edit") { - const toDeleteTransformations = _.compact(_.map(transformation_config, config => { - if (_.includes(["update", "remove"], _.get(config, "action"))) { - return _.get(config, ["value", "field_key"]) - } - })) - const updatedExistingTransformations = _.compact(_.map(existingTransformations, configs => { - if (!_.includes(toDeleteTransformations, _.get(configs, "field_key"))) { - return configs - } - })) || [] - const newTransformations = _.compact(_.map(transformation_config, config => { - if (_.includes(["update", "add"], _.get(config, "action"))) { - return config - } - })) || [] - resultantTransformations = [...updatedExistingTransformations, ...newTransformations] - } - if (action === "create") { - resultantTransformations = transformation_config || [] - } + const { id, denorm_config, data_schema, dataset_config } = configs + const indexCol = _.get(dataset_config, "timestamp_key") + const datasetTransformations = await DatasetTransformationsDraft.findAll({ where: { dataset_id: id }, raw: true }) let denormFields = _.get(denorm_config, "denorm_fields") let updatedColumns = flattenSchema(data_schema) - const transformedFields = _.filter(resultantTransformations, field => _.get(field, ["metadata", "section"]) === "transformation") - let additionalFields = _.filter(resultantTransformations, field => _.get(field, ["metadata", "section"]) === "additionalFields") + const transformedFields = _.filter(datasetTransformations, field => _.get(field, ["metadata", "section"]) === "transformation") + let additionalFields = _.filter(datasetTransformations, field => _.get(field, ["metadata", "section"]) === "additionalFields") updatedColumns = _.map(updatedColumns, (item) => { - const transformedData = _.find(transformedFields, { field_key: item.column }); + const transformedData: any = _.find(transformedFields, { field_key: item.column }); if (transformedData) { const data = _.get(transformedData, "metadata") return { diff --git a/api-service/src/v2/services/IngestionService.ts b/api-service/src/v2/services/IngestionService.ts index 9ab64de6..849d14fb 100644 --- a/api-service/src/v2/services/IngestionService.ts +++ b/api-service/src/v2/services/IngestionService.ts @@ -33,7 +33,8 @@ const connectorInstanceSpecObj = { } export const generateIngestionSpec = (payload: Record) => { - const { indexCol = defaultIndexCol, data_schema, id, dataset_id } = payload + const { data_schema, id, dataset_id, dataset_config } = payload + const indexCol = _.get(dataset_config, "timestamp_key") || defaultIndexCol const isValidTimestamp = checkTimestampCol({ indexCol, data_schema }) if (!isValidTimestamp) { throw { @@ -44,7 +45,7 @@ export const generateIngestionSpec = (payload: Record) => { } as ErrorObject } const simplifiedSpec = generateExpression(_.get(data_schema, "properties"), indexCol); - const generatedSpec = process(simplifiedSpec, indexCol) + const generatedSpec = process(simplifiedSpec) const ingestionTemplate = generateIngestionTemplate({ generatedSpec, id, indexCol, dataset_id, type: "druid" }) return ingestionTemplate } @@ -69,26 +70,18 @@ const checkTimestampCol = (schema: Record) => { return true } -const process = (spec: Map, indexCol: string): IngestionSpecModel => { +const process = (spec: Map): IngestionSpecModel => { const colValues = Array.from(spec.values()) const dimensions = filterDimensionCols(colValues) return { "dimensions": getObjByKey(dimensions, "dimensions"), "metrics": filterMetricsCols(spec), - "flattenSpec": filterFlattenSpec(colValues, indexCol) + "flattenSpec": filterFlattenSpec(colValues) } } -const filterFlattenSpec = (column: Record, indexCol: string) => { - let flattenSpec = getObjByKey(column, "flattenSpec") - if (indexCol === defaultIndexCol) { - const indexColDefaultSpec = { - "expr": ingestionConfig.syncts_path, - "name": ingestionConfig.indexCol["Event Arrival Time"], - "type": "path" - } - flattenSpec = _.concat(flattenSpec, indexColDefaultSpec) - } +const filterFlattenSpec = (column: Record) => { + const flattenSpec = getObjByKey(column, "flattenSpec") return flattenSpec } @@ -219,6 +212,7 @@ export const getDruidIngestionTemplate = (payload: Record) => { "timestampSpec": { "column": indexCol, "format": "auto" }, "metricsSpec": metrics, "granularitySpec": getGranularityObj(), + "transformSpec": {} }, "tuningConfig": { "type": "kafka", diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index b2dbb7f3..d24bbe84 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -44,7 +44,8 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, @@ -325,7 +326,8 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, @@ -368,7 +370,8 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, @@ -420,7 +423,8 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, @@ -510,11 +514,13 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 }, { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] } diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts index a905b1c0..1ff92243 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts @@ -47,7 +47,8 @@ export const TestInputsForDatasetList = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, @@ -121,7 +122,8 @@ export const TestInputsForDatasetList = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts index 93c118e8..2760d116 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -47,7 +47,8 @@ export const TestInputsForDatasetRead = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, @@ -122,7 +123,8 @@ export const TestInputsForDatasetRead = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 } ] }, diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 02ad0980..7c6122ba 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -10,6 +10,8 @@ import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { commandHttpService } from "../../../connections/commandServiceConnection"; import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; chai.use(spies); chai.should(); @@ -25,7 +27,13 @@ describe("DATASET STATUS TRANSITION LIVE", () => { it("Dataset status transition success: When the action is to set dataset live", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "ReadyToPublish" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_READ) + }) + chai.spy.on(DatasetTransformationsDraft, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(DatasourceDraft, "upsert", () => { + return Promise.resolve({}) }) chai.spy.on(commandHttpService, "post", () => { return Promise.resolve({}) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index 7deb12f7..866ef819 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -1,187 +1,219 @@ export const TestInputsForDatasetStatusTransition = { - VALID_SCHEMA_FOR_DELETE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Delete" - } + VALID_SCHEMA_FOR_DELETE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_SCHEMA_FOR_LIVE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Live" - } + "request": { + "dataset_id": "telemetry.1", + "status": "Delete" + } + }, + VALID_SCHEMA_FOR_LIVE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_SCHEMA_FOR_RETIRE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry", - "status": "Retire" - } + "request": { + "dataset_id": "telemetry.1", + "status": "Live" + } + }, + VALID_SCHEMA_FOR_RETIRE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - INVALID_SCHEMA: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "" + "request": { + "dataset_id": "telemetry", + "status": "Retire" + } + }, + INVALID_SCHEMA: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry.1", + "status": "" + } + }, + VALID_REQUEST_FOR_READY_FOR_PUBLISH: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry.1", + "status": "ReadyToPublish" + } + }, + VALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "dataset_id": "telemetry", + "type": "dataset", + "name": "sb-telemetry", + "id": "telemetry.1", + "status": "Draft", + "version_key": "1789887878", + "validation_config": { + "validate": true, + "mode": "Strict" + }, + "extraction_config": { + "is_batch_event": true, + "extraction_key": "events", + "batch_id": "id", + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "id", + "dedup_period": 3783 + } + }, + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "mid", + "dedup_period": 3783 + }, + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ets": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "required": [ + "eid" + ] + }, + "additionalProperties": true + }, + "router_config": { + "topic": "test" + }, + "denorm_config": { + "redis_db_host": "local", + "redis_db_port": 5432, + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "dataset_name": "name", + "dataset_id": "name", + "redis_db": 10 + }, + { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_name": "name", + "dataset_id": "name", + "redis_db": 15 } + ] }, - VALID_REQUEST_FOR_READY_FOR_PUBLISH:{ - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "ReadyToPublish" + "dataset_config": { + "data_key": "mid", + "timestamp_key": "ets", + "entry_topic": "topic", + "redis_db_host": "local", + "redis_db_port": 5432, + "redis_db": 0, + "index_data": true + }, + "client_state": {}, + "tags": [ + "tag1", + "tag2" + ] + }, + INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "dataset_id": "telemetry", + "type": "", + "name": "sb-telemetry", + "id": "telemetry.1", + "status": "Draft", + "version_key": "1789887878", + "validation_config": { + "validate": true, + "mode": "Strict" + }, + "router_config": { + "topic": "test" + }, + "denorm_config": { + "redis_db_host": "local", + "redis_db_port": 5432, + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "dataset_name": "name", + "dataset_id": "name", + "redis_db": 15 + }, + { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_name": "name", + "dataset_id": "name", + "redis_db": 23 } + ] }, - VALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "dataset", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "batch_id": "id", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 3783 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 3783 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "ets": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" + "dataset_config": { + "data_key": "mid", + "timestamp_key": "ets", + "entry_topic": "topic", + "redis_db_host": "local", + "redis_db_port": 5432, + "redis_db": 0, + "index_data": true + }, + "client_state": {}, + "tags": [ + "tag1", + "tag2" + ] + }, + + VALID_SCHEMA_FOR_LIVE_READ: { + "dataset_id": "sb-ddd", + "type": "dataset", + "name": "sb-telemetry2", + "status": "ReadyToPublish", + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ets": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "required": [ + "eid" ] }, - INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - } + "additionalProperties": true + }, + "dataset_config": { + "data_key": "", + "timestamp_key": "ets" + }, + "tags": [] + } } \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index 831cd534..94d11681 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -90,7 +90,8 @@ describe("DATASET DENORM UPDATE", () => { id: "telemetry", version_key: validVersionKey, type:"dataset", status: "Draft", denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "redis_db": 10 }] } }) @@ -148,7 +149,8 @@ describe("DATASET DENORM UPDATE", () => { id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 }] } }) @@ -181,7 +183,8 @@ describe("DATASET DENORM UPDATE", () => { id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "id" + "denorm_out_field": "id", + "redis_db": 10 }] } }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index cb1fd6be..e90be894 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -183,7 +183,8 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "redis_db": 10 }] } }) @@ -219,7 +220,8 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { id: "telemetry", status: "Draft", type:"dataset" , version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "redis_db": 10 }] } }) @@ -255,7 +257,8 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "redis_db": 10 }] } }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 0ee5b588..56c534ae 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -59,7 +59,8 @@ describe("DATASET UPDATE API", () => { id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "redis_db": 10 }] } }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 4dd1455e..c9217687 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -58,7 +58,8 @@ export const TestInputsForDatasetUpdate = { { "values": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 }, "action": "add" } @@ -93,7 +94,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -212,7 +217,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -229,7 +238,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -284,14 +297,16 @@ export const TestInputsForDatasetUpdate = { { "values": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 }, "action": "add" }, { "values": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "redis_db": 10 }, "action": "remove" } @@ -301,7 +316,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -310,7 +329,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key2", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -319,7 +342,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key3", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -359,14 +386,16 @@ export const TestInputsForDatasetUpdate = { { "values": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 }, "action": "add" }, { "values": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "redis_db": 10 }, "action": "add" } @@ -434,7 +463,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -443,7 +476,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key1", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -452,7 +489,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key2", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -461,7 +502,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key2", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -470,7 +515,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key3", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, @@ -479,7 +528,11 @@ export const TestInputsForDatasetUpdate = { { "values": { "field_key": "key3", - "transformation_function": {}, + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, "mode": "Strict", "metadata": {} }, From 72171d365f9128484c4544d1051c37815741834e Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 12:56:40 +0530 Subject: [PATCH 015/311] Sanketika-obsrv/issue-tracker: #OBS-150: checking type and setting value --- api-service/src/v2/services/CloudServices/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/v2/services/CloudServices/index.ts b/api-service/src/v2/services/CloudServices/index.ts index 6245ad19..c458016d 100644 --- a/api-service/src/v2/services/CloudServices/index.ts +++ b/api-service/src/v2/services/CloudServices/index.ts @@ -5,7 +5,8 @@ import { GCPStorageService } from "./GCPStorageService"; import * as _ from "lodash"; const cloudProviderName = _.get(config, "cloud_config.cloud_storage_provider"); -const cloudProviderConfig = _.get(config, "cloud_config.cloud_storage_config"); +const cloudConfig = _.get(config, "cloud_config.cloud_storage_config"); +const cloudProviderConfig = _.isString(cloudConfig) ? JSON.parse(cloudConfig) : cloudConfig; const initialiseServiceProvider = (provider: any, config: any): AzureStorageService | AWSStorageService | GCPStorageService => { switch (provider) { From 6d9f8e8224614b8b79273c1cc38774ead8b00cea Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 13:11:19 +0530 Subject: [PATCH 016/311] Sanketika-obsrv/issue-tracker: #OBS-150: chnaged config to defaults --- api-service/src/v2/configs/Config.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index afabc4ca..b30783cb 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -74,11 +74,11 @@ export const config = { } }, "cloud_config": { - "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure - "cloud_storage_region": process.env.cloud_storage_region || "us-east-2", // Region for the cloud provider storage - "cloud_storage_config": process.env.cloud_storage_config || {}, // Respective credentials object for cloud provider. Optional if service account provided - "container": process.env.container || "exhaust-test-bucket", // Storage container/bucket name - "container_prefix": process.env.container_prefix || "connector-registry", // Path to the folder inside container/bucket. Empty if data at root level + "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure + "cloud_storage_region": process.env.cloud_storage_region || "", // Region for the cloud provider storage + "cloud_storage_config": process.env.cloud_storage_config ? JSON.parse(process.env.cloud_storage_config) : {}, // Respective credentials object for cloud provider. Optional if service account provided + "container": process.env.container || "container", // Storage container/bucket name + "container_prefix": process.env.container_prefix || "", // Path to the folder inside container/bucket. Empty if data at root level "storage_url_expiry": process.env.storage_url_expiry ? parseInt(process.env.storage_url_expiry) : 3600, // in seconds, Default 1hr of expiry for Signed URLs. "maxQueryDateRange": process.env.exhaust_query_range ? parseInt(process.env.exhaust_query_range) : 31, // in days. Defines the maximum no. of days the files can be fetched "exclude_exhaust_types": process.env.exclude_exhaust_types ? process.env.exclude_exhaust_types.split(",") : ["system-stats", "masterdata-system-stats", "system-events",] // list of folder type names to skip exhaust service @@ -95,7 +95,7 @@ export const config = { }, "command_service_config": { "host": process.env.command_service_host || "http://localhost", - "port": parseInt(process.env.command_service_port || "9999"), + "port": parseInt(process.env.command_service_port || "8000"), "path": process.env.command_service_path || "/system/v1/dataset/command", "connector_registry_path": process.env.connector_registry_path || "/connector/v1/register", }, From 2dbed92802c918ef8eaf9877f7e01b690835fdca Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 13:13:08 +0530 Subject: [PATCH 017/311] Sanketika-obsrv/issue-tracker: #OBS-150: chnaged config to defaults --- api-service/src/v2/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index b30783cb..3ec3d345 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -74,7 +74,7 @@ export const config = { } }, "cloud_config": { - "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure + "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure "cloud_storage_region": process.env.cloud_storage_region || "", // Region for the cloud provider storage "cloud_storage_config": process.env.cloud_storage_config ? JSON.parse(process.env.cloud_storage_config) : {}, // Respective credentials object for cloud provider. Optional if service account provided "container": process.env.container || "container", // Storage container/bucket name From 4c911b8bc76fc64a0f1c62d839c11949193b3e35 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 15:17:30 +0530 Subject: [PATCH 018/311] Sanketika-obsrv/issue-tracker: #OBS-150: removed path dependency --- api-service/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/package.json b/api-service/package.json index d7b15957..c7d2370b 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -44,7 +44,6 @@ "moment": "^2.29.4", "multiparty": "4.2.1", "node-sql-parser": "^5.1.0", - "path": "^0.12.7", "pg": "^8.11.3", "pg-hstore": "^2.3.4", "prom-client": "^14.2.0", From 28c00a11e7e0278f504b6e0ab6cec08038147173 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 26 Jun 2024 15:42:34 +0530 Subject: [PATCH 019/311] Sanketika-Obsrv/issue-tracker#OBS-151: feat: set redis db value for denorm fields using dataset_id --- .../DatasetCreateValidationSchema.json | 7 ++-- .../DatasetUpdateValidationSchema.json | 7 ++-- api-service/src/v2/services/DatasetService.ts | 26 +++++++++++- .../src/v2/services/IngestionService.ts | 3 +- .../DatasetCreate/DatasetCreate.spec.ts | 42 ++++++++++++++++--- .../DatasetCreate/Fixtures.ts | 12 +++--- .../DatasetManagement/DatasetList/Fixtures.ts | 4 +- .../DatasetManagement/DatasetRead/Fixtures.ts | 4 +- .../DatasetStatusTransition/Fixtures.ts | 12 ++---- .../DatasetUpdate/DatasetDenorm.spec.ts | 38 ++++++++++++++--- .../DatasetTransformation.spec.ts | 13 ++++++ .../DatasetUpdate/DatasetUpdate.spec.ts | 4 ++ .../DatasetUpdate/Fixtures.ts | 10 ++--- 13 files changed, 138 insertions(+), 44 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index a8715500..b0d8780a 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -131,11 +131,12 @@ "type": "string", "minLength": 1 }, - "redis_db": { - "type": "number" + "dataset_id": { + "type": "string", + "minLength": 1 } }, - "required": ["denorm_key", "denorm_out_field", "redis_db"], + "required": ["denorm_key", "denorm_out_field", "dataset_id"], "additionalProperties": false } } diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index b20c0a51..c11ee4c6 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -316,8 +316,9 @@ "type": "string", "minLength": 1 }, - "redis_db": { - "type": "number" + "dataset_id": { + "type": "string", + "minLength": 1 } }, "required": ["denorm_out_field"] @@ -339,7 +340,7 @@ "then": { "properties": { "values": { - "required": ["denorm_key", "redis_db"] + "required": ["denorm_key", "dataset_id"] } } } diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 8a8b5c81..e7b4c6ea 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -10,6 +10,7 @@ import { getUpdatedSchema } from "./DatasourceService"; import { DatasetType } from "../types/DatasetModels"; import { defaultDatasetConfig, defaultMasterConfig } from "../configs/DatasetConfigDefault"; import { query } from "../connections/databaseConnection"; +import { ErrorObject } from "../types/ResponseModel"; export const getDataset = async (datasetId: string, raw = false): Promise => { const dataset = await Dataset.findOne({ @@ -76,7 +77,8 @@ const getDataSource = (ingestionPayload: Record) => { const getDatasetDefaults = async (payload: Record): Promise> => { const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) - return datasetPayload + const denormPayload = await updateDenormFields(_.get(datasetPayload, "denorm_config")) + return { ...datasetPayload, denorm_config: denormPayload } } const setRedisDBConfig = async (datasetConfig: Record): Promise> => { @@ -115,4 +117,26 @@ const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: const defaults = _.cloneDeep(defaultConfig) const datasetConfigs = _.merge(defaults, modifyPayload) return datasetConfigs +} + +const updateDenormFields = async (denormConfigs: Record) => { + const denormFields = _.get(denormConfigs, "denorm_fields") + if (_.isEmpty(denormFields)) { + return denormConfigs + } + const masterDatasets = await Dataset.findAll({ where: { type: DatasetType.MasterDataset }, raw: true }); + const updatedFields = _.map(denormFields, fields => { + const master = _.find(masterDatasets, dataset => _.get(dataset, ["dataset_id"]) === fields.dataset_id) + if (_.isEmpty(master)) { + throw { + code: "DATASET_DENORM_NOT_FOUND", + message: "Denorm Master dataset not found", + statusCode: 404, + errCode: "NOT_FOUND" + } as ErrorObject + } + const redis_db = _.get(master, ["dataset_config", "redis_db"]) + return { ...fields, redis_db } + }) + return { ...denormConfigs, denorm_fields: updatedFields } } \ No newline at end of file diff --git a/api-service/src/v2/services/IngestionService.ts b/api-service/src/v2/services/IngestionService.ts index 849d14fb..4a4015f7 100644 --- a/api-service/src/v2/services/IngestionService.ts +++ b/api-service/src/v2/services/IngestionService.ts @@ -211,8 +211,7 @@ export const getDruidIngestionTemplate = (payload: Record) => { "dimensionsSpec": { "dimensions": dimensions }, "timestampSpec": { "column": indexCol, "format": "auto" }, "metricsSpec": metrics, - "granularitySpec": getGranularityObj(), - "transformSpec": {} + "granularitySpec": getGranularityObj() }, "tuningConfig": { "type": "kafka", diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index d70c7d1f..8dbec85c 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -37,13 +37,19 @@ describe("DATASET CREATE API", () => { return Promise.resolve({ dataValues: { id: "telemetry" } }) }) chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "data_schema": {"$schema": "https://json-schema.org/draft/2020-12/schema","type": "object", - "properties": { - "eid": {"type": "string"}, - "ets": {"type": "string"} + return Promise.resolve({ + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", + "properties": { + "eid": { "type": "string" }, + "ets": { "type": "string" } + }, + "additionalProperties": true }, - "additionalProperties": true - },}) + }) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master-telemetry", "dataset_config": { "redis_db": 15 } }]) }) chai.spy.on(DatasetTransformationsDraft, "findAll", () => { return Promise.resolve() @@ -156,4 +162,28 @@ describe("DATASET CREATE API", () => { }); }); + it("Failure: Master dataset not found as denorm", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(null) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "trip-data-master", "dataset_config": { "redis_db": 15 } }]) + }) + + chai + .request(app) + .post("/v2/datasets/create") + .send(TestInputsForDatasetCreate.VALID_DATASET) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Denorm Master dataset not found") + res.body.error.code.should.be.eq("DATASET_DENORM_NOT_FOUND") + done(); + }); + }); + }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index d24bbe84..54edb673 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -45,7 +45,7 @@ export const TestInputsForDatasetCreate = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, @@ -327,7 +327,7 @@ export const TestInputsForDatasetCreate = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, @@ -371,7 +371,7 @@ export const TestInputsForDatasetCreate = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, @@ -424,7 +424,7 @@ export const TestInputsForDatasetCreate = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, @@ -515,12 +515,12 @@ export const TestInputsForDatasetCreate = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" }, { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] } diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts index 1ff92243..a6d9dd9b 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts @@ -48,7 +48,7 @@ export const TestInputsForDatasetList = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, @@ -123,7 +123,7 @@ export const TestInputsForDatasetList = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts index 2760d116..7ae37c97 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -48,7 +48,7 @@ export const TestInputsForDatasetRead = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, @@ -124,7 +124,7 @@ export const TestInputsForDatasetRead = { { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" } ] }, diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index 866ef819..ee8e9dee 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -112,15 +112,13 @@ export const TestInputsForDatasetStatusTransition = { "denorm_key": "actor.id", "denorm_out_field": "userdata", "dataset_name": "name", - "dataset_id": "name", - "redis_db": 10 + "dataset_id": "name" }, { "denorm_key": "actor.id", "denorm_out_field": "mid", "dataset_name": "name", - "dataset_id": "name", - "redis_db": 15 + "dataset_id": "name" } ] }, @@ -161,15 +159,13 @@ export const TestInputsForDatasetStatusTransition = { "denorm_key": "actor.id", "denorm_out_field": "userdata", "dataset_name": "name", - "dataset_id": "name", - "redis_db": 15 + "dataset_id": "name" }, { "denorm_key": "actor.id", "denorm_out_field": "mid", "dataset_name": "name", - "dataset_id": "name", - "redis_db": 23 + "dataset_id": "name" } ] }, diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index 94d11681..e1e08e83 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -9,6 +9,7 @@ import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" import { sequelize } from "../../../connections/databaseConnection"; +import { Dataset } from "../../../models/Dataset"; chai.use(spies); chai.should(); @@ -29,6 +30,9 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master-telemetry", "dataset_config": { "redis_db": 15 } }]) + }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) @@ -91,6 +95,7 @@ describe("DATASET DENORM UPDATE", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } @@ -150,17 +155,12 @@ describe("DATASET DENORM UPDATE", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "userdata", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -184,6 +184,7 @@ describe("DATASET DENORM UPDATE", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "id", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } @@ -204,4 +205,29 @@ describe("DATASET DENORM UPDATE", () => { done(); }); }); + + it("Failure: Master dataset not found as denorm", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, denorm_config: { denorm_field: [] } + }) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "trip-events", "dataset_config": { "redis_db": 15 } }]) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DENORM_ADD) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Denorm Master dataset not found") + res.body.error.code.should.be.eq("DATASET_DENORM_NOT_FOUND") + done(); + }); + }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index e90be894..6dd90d98 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -10,6 +10,7 @@ import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" import { sequelize } from "../../../connections/databaseConnection"; +import { Dataset } from "../../../models/Dataset"; chai.use(spies); chai.should(); @@ -184,6 +185,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } @@ -192,6 +194,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetTransformationsDraft, "findAll", () => { return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master-telemetry", "dataset_config": { "redis_db": 15 } }]) + }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) @@ -221,6 +226,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } @@ -229,6 +235,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetTransformationsDraft, "findAll", () => { return Promise.resolve([{ field_key: "key7" }, { field_key: "key2" }]) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master-telemetry", "dataset_config": { "redis_db": 15 } }]) + }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) @@ -258,6 +267,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } @@ -266,6 +276,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetTransformationsDraft, "findAll", () => { return Promise.resolve([{ field_key: "key7" }, { field_key: "key3" }]) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master-telemetry", "dataset_config": { "redis_db": 15 } }]) + }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 56c534ae..34f62a7c 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -60,6 +60,7 @@ describe("DATASET UPDATE API", () => { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid", + "dataset_id" : "master-telemetry", "redis_db": 10 }] } @@ -77,6 +78,9 @@ describe("DATASET UPDATE API", () => { }, }) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master-telemetry", "dataset_config": { "redis_db": 15 } }]) + }) chai.spy.on(DatasetTransformations, "findAll", () => { return Promise.resolve() }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index c9217687..13198739 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -59,7 +59,7 @@ export const TestInputsForDatasetUpdate = { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" }, "action": "add" } @@ -298,7 +298,7 @@ export const TestInputsForDatasetUpdate = { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" }, "action": "add" }, @@ -306,7 +306,7 @@ export const TestInputsForDatasetUpdate = { "values": { "denorm_key": "actor.id", "denorm_out_field": "mid", - "redis_db": 10 + "dataset_id" : "master-telemetry" }, "action": "remove" } @@ -387,7 +387,7 @@ export const TestInputsForDatasetUpdate = { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" }, "action": "add" }, @@ -395,7 +395,7 @@ export const TestInputsForDatasetUpdate = { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata", - "redis_db": 10 + "dataset_id" : "master-telemetry" }, "action": "add" } From 2000ca6571431a662717110fa2327485fc0c3a67 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 26 Jun 2024 15:47:35 +0530 Subject: [PATCH 020/311] Sanketika-Obsrv/issue-tracker#OBS-151: fix: Rename fix --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 818c360d..b902006a 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -158,7 +158,7 @@ const deleteDraftRecords = async (config: Record) => { } //GENERATE_INGESTION_SPEC -const generateDatasource = async (config: Record) => { +const generateIngestionSpec = async (config: Record) => { const { dataset } = config; const dataSource = await generateDataSource(dataset); return DatasourceDraft.upsert(dataSource) @@ -231,7 +231,7 @@ const restartPipeline = async (config: Record) => { const commandExecutors = { DELETE_DRAFT_DATASETS: deleteDataset, PUBLISH_DATASET: publishDataset, - GENERATE_INGESTION_SPEC: generateDatasource, + GENERATE_INGESTION_SPEC: generateIngestionSpec, CHECK_DATASET_IS_DENORM: checkDatasetDenorm, SET_DATASET_TO_RETIRE: setDatasetRetired, DELETE_SUPERVISORS: deleteSupervisors, From b30bcc916885b91dc588699e3e4644e5f5ab0f97 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 26 Jun 2024 16:56:31 +0530 Subject: [PATCH 021/311] Sanketika-Obsrv/issue-tracker#OBS-151: feat: update dataset schema fix --- .../DatasetUpdate/DatasetUpdateValidationSchema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index c11ee4c6..468a43ed 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -321,7 +321,8 @@ "minLength": 1 } }, - "required": ["denorm_out_field"] + "required": ["denorm_out_field"], + "additionalProperties": false }, "action": { "type": "string", From f3b105ed2e39bf7ab4dbf9eff30203495fe25f10 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 17:37:34 +0530 Subject: [PATCH 022/311] Sanketika-obsrv/issue-tracker: #OBS-150: changes in config file --- api-service/src/v2/configs/Config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 3ec3d345..81ef91e0 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -95,13 +95,13 @@ export const config = { }, "command_service_config": { "host": process.env.command_service_host || "http://localhost", - "port": parseInt(process.env.command_service_port || "8000"), + "port": parseInt(process.env.command_service_port || "9999"), "path": process.env.command_service_path || "/system/v1/dataset/command", "connector_registry_path": process.env.connector_registry_path || "/connector/v1/register", }, "obsrv_api_service_config": { - "host": process.env.obser_api_service_host || "http://localhost", - "port": parseInt(process.env.obser_api_service_port || "3000"), + "host": process.env.obsrv_api_service_host || "http://localhost", + "port": parseInt(process.env.obsrv_api_service_port || "3000"), "generate_url_path": process.env.generate_url_path || "/v2/files/generate-url", "path": process.env.command_service_path || "/system/v1/dataset/command" }, From f95167dd26796e589ed1f2f82f519fc8ae1e240e Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 17:38:37 +0530 Subject: [PATCH 023/311] Sanketika-obsrv/issue-tracker: #OBS-150: changes in config file --- api-service/src/v2/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 81ef91e0..867f26fb 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -95,7 +95,7 @@ export const config = { }, "command_service_config": { "host": process.env.command_service_host || "http://localhost", - "port": parseInt(process.env.command_service_port || "9999"), + "port": parseInt(process.env.command_service_port || "8000"), "path": process.env.command_service_path || "/system/v1/dataset/command", "connector_registry_path": process.env.connector_registry_path || "/connector/v1/register", }, From 685e0729a13e080014dffa0267335ac84af2165e Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 17:46:21 +0530 Subject: [PATCH 024/311] Sanketika-obsrv/issue-tracker: #OBS-150: updated api id --- .../ConnectorRegistryStreamController.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 941e9dfb..8bda5527 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -11,7 +11,7 @@ import { PassThrough } from "stream"; import { URLAccess } from "../../types/SampleURLModel"; import { ErrorObject } from "../../types/ResponseModel"; -export const apiId = "api.files.generate-url"; +export const apiId = "api.connector.stream.upload"; export const code = "FAILED_TO_REGISTER_CONNECTOR"; const apiServiceHost = _.get(config, ["obsrv_api_service_config", "host"]); From 2c216fea140fa5e0c798cf09b7fcb2261b616843 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 17:48:07 +0530 Subject: [PATCH 025/311] Sanketika-obsrv/issue-tracker: #OBS-150: added default ts in request body --- .../ConnectorRegistryStreamController.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 8bda5527..1f5c8cfe 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -25,7 +25,7 @@ const registryUrl = _.get(config, ["command_service_config", "connector_registry const getGenerateSignedURLRequestBody = (files: string[], access: string) => ({ id: apiId, ver: "v2", - ts: new Date().toISOString(), + ts: Date.now(), params: { msgid: v4() }, From d7d94cadb7b39456d8e376a76b5a6fcd83edcf95 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 26 Jun 2024 18:09:16 +0530 Subject: [PATCH 026/311] Sanketika-obsrv/issue-tracker: #OBS-150: added default ts in request body --- .../ConnectorRegistryStreamController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 1f5c8cfe..1ee82751 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -23,9 +23,9 @@ const commandServicePort = _.get(config, ["command_service_config", "port"]); const registryUrl = _.get(config, ["command_service_config", "connector_registry_path"]) const getGenerateSignedURLRequestBody = (files: string[], access: string) => ({ - id: apiId, + id: "api.files.generate-url", ver: "v2", - ts: Date.now(), + ts: Date.now().toString(), params: { msgid: v4() }, From 04539fdf4cee3ea95af4bd29c30a79ee95f77a2d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 28 Jun 2024 11:17:27 +0530 Subject: [PATCH 027/311] Sanketika-Obsrv/issue-tracker#OBS-151: dataset create and update schema changes to support files_upload_path prop --- .../DatasetCreateValidationSchema.json | 16 ++++------------ .../DatasetUpdateValidationSchema.json | 2 +- api-service/swagger-doc/openapi_v2.yml | 4 ++++ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index b0d8780a..abb0080c 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -154,7 +154,7 @@ "type": "string", "minLength": 1 }, - "file_upload_path": { + "files_upload_path": { "type": "array", "items": { "type": "string", @@ -162,7 +162,6 @@ } } }, - "required": ["data_key", "timestamp_key"], "additionalProperties": false }, "transformations_config": { @@ -201,21 +200,14 @@ }, "mode": { "type": "string", - "enum": [ - "Strict", - "Lenient" - ] + "enum": ["Strict", "Lenient"] }, "metadata": { "type": "object" } }, "additionalProperties": false, - "required": [ - "field_key", - "transformation_function", - "mode" - ] + "required": ["field_key", "transformation_function", "mode"] } }, "tags": { @@ -232,4 +224,4 @@ }, "required": ["id", "ver", "ts", "params", "request"], "additionalProperties": false -} \ No newline at end of file +} diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 468a43ed..4389dd05 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -195,7 +195,7 @@ "dataMappings": { "type": "object" }, - "file_upload_path": { + "files_upload_path": { "type": "array", "items": { "type": "string", diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 8814daac..36edef47 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -78,6 +78,7 @@ paths: denorm_fields: - denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry transformations_config: - field_key: eid transformation_function: @@ -93,6 +94,7 @@ paths: dataset_config: data_key: mid timestamp_key: ets + files_upload_path: ["telemetry.json"] tags: - tag1 - tag2 @@ -261,10 +263,12 @@ paths: - values: denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry action: remove - values: denorm_key: eid denorm_out_field: edata + dataset_id: trip-record action: add transformation_config: - values: From 4f3cac0aee9346a9ea02e6b0b91c1a876b7b5c77 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 28 Jun 2024 11:17:56 +0530 Subject: [PATCH 028/311] Sanketika-Obsrv/issue-tracker#OBS-151: configs fix --- api-service/src/v2/configs/DatasetConfigDefault.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/configs/DatasetConfigDefault.ts b/api-service/src/v2/configs/DatasetConfigDefault.ts index 850d0a05..a59c88ee 100644 --- a/api-service/src/v2/configs/DatasetConfigDefault.ts +++ b/api-service/src/v2/configs/DatasetConfigDefault.ts @@ -38,7 +38,7 @@ export const defaultMasterConfig = { "redis_db_port": config.redis_config.denorm_redis_port, "index_data": true, "redis_db": 3, - "file_upload_path": [] + "files_upload_path": [] }, "status": DatasetStatus.Draft, "version": 1, @@ -82,7 +82,7 @@ export const defaultDatasetConfig = { "redis_db_port": config.redis_config.dedup_redis_port, "index_data": true, "redis_db": 0, - "file_upload_path": [] + "files_upload_path": [] }, "status": DatasetStatus.Draft, "api_version": "v2", From 7601bf653a63972129cd03437d0538cb4f02e5ef Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 28 Jun 2024 11:27:31 +0530 Subject: [PATCH 029/311] Sanketika-Obsrv/issue-tracker#OBS-151: test case fixes --- .../DatasetStatusTransition/ReadyToPublishSchema.json | 2 +- .../v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts | 6 +++--- .../v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts | 4 ++-- api-service/swagger-doc/openapi_v2.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 743b2944..8ffd9591 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -260,7 +260,7 @@ "redis_db": { "type": "integer" }, - "file_upload_path": { + "files_upload_path": { "type": "array", "items": { "type": "string", diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index 54edb673..97a195bb 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -52,7 +52,7 @@ export const TestInputsForDatasetCreate = { "dataset_config": { "data_key": "", "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "files_upload_path": ["/config/file.json"] }, "tags": [] } @@ -126,7 +126,7 @@ export const TestInputsForDatasetCreate = { "dataset_config": { "data_key": "", "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "files_upload_path": ["/config/file.json"] }, "transformations_config": [ { @@ -182,7 +182,7 @@ export const TestInputsForDatasetCreate = { "dataset_config": { "data_key": "", "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "files_upload_path": ["/config/file.json"] }, "transformations_config": [ { diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 13198739..33fd4640 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -193,7 +193,7 @@ export const TestInputsForDatasetUpdate = { "dataset_config": { "data_key": "eid", "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "files_upload_path": ["/config/file.json"] } } }, @@ -356,7 +356,7 @@ export const TestInputsForDatasetUpdate = { "dataset_config": { "data_key": "mid", "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "files_upload_path": ["/config/file.json"] }, "tags": [ { diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 36edef47..d93d8bf4 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -979,7 +979,7 @@ paths: redis_db_port: 6379 index_data: true redis_db: 70 - file_upload_path: [] + files_upload_path: [] configurations: indexConfiguration: index: From 422a80c3b4caf979e626e5bcfdc720897c9ee8b9 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 15:18:42 +0530 Subject: [PATCH 030/311] Sanketika-obsrv/issue-tracker: #OBS-150: removed config --- api-service/src/v2/configs/Config.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 867f26fb..60a048c1 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -99,12 +99,6 @@ export const config = { "path": process.env.command_service_path || "/system/v1/dataset/command", "connector_registry_path": process.env.connector_registry_path || "/connector/v1/register", }, - "obsrv_api_service_config": { - "host": process.env.obsrv_api_service_host || "http://localhost", - "port": parseInt(process.env.obsrv_api_service_port || "3000"), - "generate_url_path": process.env.generate_url_path || "/v2/files/generate-url", - "path": process.env.command_service_path || "/system/v1/dataset/command" - }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", "masterdata_processor_job_manager_url": process.env.masterdata_processor_job_manager_url || "http://localhost:8081" From 7091aec6b8576ffd269e2dc3f7661003b9cbe91f Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 15:20:02 +0530 Subject: [PATCH 031/311] Sanketika-obsrv/issue-tracker: #OBS-150: generalized generate signed url method --- .../ConnectorRegistryStreamController.ts | 79 +++++++------------ .../GenerateSignedURL/GenerateSignedURL.ts | 66 +--------------- .../controllers/GenerateSignedURL/helper.ts | 64 +++++++++++++++ 3 files changed, 94 insertions(+), 115 deletions(-) create mode 100644 api-service/src/v2/controllers/GenerateSignedURL/helper.ts diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index 1ee82751..dc66d595 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -6,38 +6,19 @@ import { config } from "../../configs/Config"; import axios from "axios"; import httpStatus from "http-status"; import busboy from "busboy"; -import { v4 } from "uuid" import { PassThrough } from "stream"; -import { URLAccess } from "../../types/SampleURLModel"; -import { ErrorObject } from "../../types/ResponseModel"; +import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; export const apiId = "api.connector.stream.upload"; export const code = "FAILED_TO_REGISTER_CONNECTOR"; -const apiServiceHost = _.get(config, ["obsrv_api_service_config", "host"]); -const apiServicePort = _.get(config, ["obsrv_api_service_config", "port"]); -const generateSignedURLPath = _.get(config, ["obsrv_api_service_config", "generate_url_path"]); - const commandServiceHost = _.get(config, ["command_service_config", "host"]); const commandServicePort = _.get(config, ["command_service_config", "port"]); const registryUrl = _.get(config, ["command_service_config", "connector_registry_path"]) - -const getGenerateSignedURLRequestBody = (files: string[], access: string) => ({ - id: "api.files.generate-url", - ver: "v2", - ts: Date.now().toString(), - params: { - msgid: v4() - }, - request: { - files, - access: access || URLAccess.Read, - type: "connector" - } -}); +let resmsgid: string | any; const connectorRegistryStream = async (req: Request, res: Response) => { - const resmsgid = _.get(res, "resmsgid"); + resmsgid = _.get(res, "resmsgid"); try { const uploadStreamResponse: any = await uploadStream(req); const registryRequestBody = { @@ -59,22 +40,6 @@ const connectorRegistryStream = async (req: Request, res: Response) => { } }; -export const generatePresignedUrl = async (fileName: string, access: string) => { - try { - const requestBody = getGenerateSignedURLRequestBody([fileName], access); - const response = await axios.post(`${apiServiceHost}:${apiServicePort}${generateSignedURLPath}`, requestBody); - return response?.data?.result; - } - catch (err) { - throw { - code: "FILES_GENERATE_URL_FAILURE", - message: "Failed to generate sample urls", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } -}; - const uploadStream = async (req: Request) => { return new Promise((resolve, reject) => { const filePromises: Promise[] = []; @@ -96,19 +61,30 @@ const uploadStream = async (req: Request) => { fileCount++; const processFile = async () => { const fileName = info?.filename; - const preSignedUrl: any = await generatePresignedUrl(fileName, URLAccess.Write); - const filePath = preSignedUrl[0]?.filePath - const fileNameExtracted = extractFileNameFromPath(filePath); - relative_path.push(...fileNameExtracted); - const pass = new PassThrough(); - file.pipe(pass); - const fileBuffer = await streamToBuffer(pass); - await axios.put(preSignedUrl[0]?.preSignedUrl, fileBuffer, { - headers: { - "Content-Type": info.mimeType, - "Content-Length": fileBuffer.length, - } - }); + try { + const preSignedUrl: any = await generatePreSignedUrl("write", [fileName], "connector") + const filePath = preSignedUrl[0]?.filePath + const fileNameExtracted = extractFileNameFromPath(filePath); + relative_path.push(...fileNameExtracted); + const pass = new PassThrough(); + file.pipe(pass); + const fileBuffer = await streamToBuffer(pass); + await axios.put(preSignedUrl[0]?.preSignedUrl, fileBuffer, { + headers: { + "Content-Type": info.mimeType, + "Content-Length": fileBuffer.length, + } + }); + } + catch (err) { + logger.error({ apiId, err, resmsgid, message: "Failed to generate sample urls", code: "FILES_GENERATE_URL_FAILURE" }) + reject({ + code: "FILES_GENERATE_URL_FAILURE", + message: "Failed to generate sample urls", + statusCode: 500, + errCode: "INTERNAL_SERVER_ERROR" + }) + } }; filePromises.push(processFile()); }); @@ -117,6 +93,7 @@ const uploadStream = async (req: Request) => { await Promise.all(filePromises); resolve(relative_path); } catch (error) { + logger.error({ apiId, error, resmsgid, message: "Fail to upload a file", code: "FAILED_TO_UPLOAD" }) reject({ code: "FAILED_TO_UPLOAD", message: "Fail to upload a file", diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts index e2bc2855..6454b1fa 100644 --- a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -6,11 +6,9 @@ import logger from "../../logger"; import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import GenerateURL from "./GenerateSignedURLValidationSchema.json" -import { cloudProvider } from "../../services/CloudServices"; import { config } from "../../configs/Config"; import { URLAccess } from "../../types/SampleURLModel"; -import { v4 as uuidv4 } from "uuid"; -import path from "path"; +import { generatePreSignedUrl } from "./helper"; export const apiId = "api.files.generate-url" export const code = "FILES_GENERATE_URL_FAILURE" @@ -48,21 +46,7 @@ const generateSignedURL = async (req: Request, res: Response) => { errCode: "BAD_REQUEST" } as ErrorObject, req, res); } - - const { filesList, updatedFileNames } = transformFileNames(files, access) - logger.info(`Updated file names with path:${updatedFileNames}`) - - const urlExpiry: number = getURLExpiry(access) - const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)) - const signedUrlList = _.map(preSignedUrls, list => { - const fileNameWithUid = _.keys(list)[0] - return { - filePath: getFilePath(fileNameWithUid), - fileName: filesList.get(fileNameWithUid), - preSignedUrl: _.values(list)[0] - } - }) - + const signedUrlList = await generatePreSignedUrl(access, files, containerType) logger.info({ apiId, requestBody, msgid, resmsgid, response: signedUrlList, message: `Sample urls generated successfully for files:${files}` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: signedUrlList }) } catch (error: any) { @@ -76,52 +60,6 @@ const generateSignedURL = async (req: Request, res: Response) => { } } -const getFilePath = (file: string) => { - const datasetUploadPath = `${config.presigned_url_configs.service}/user_uploads/${file}`; - const connectorUploadPath = `${config.cloud_config.container_prefix}/${file}`; - - const paths: Record = { - "dataset": datasetUploadPath, - "connector": connectorUploadPath - }; - - return paths[containerType] || datasetUploadPath; -} - -const transformFileNames = (fileList: Array, access: string): Record => { - if (access === URLAccess.Read) { - return transformReadFiles(fileList) - } - return transformWriteFiles(fileList) -} - -const transformReadFiles = (fileNames: Array) => { - const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { - fileMap.set(file, file) - return getFilePath(file) - }) - return { filesList: fileMap, updatedFileNames } -} - -const transformWriteFiles = (fileNames: Array) => { - const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { - const uuid = uuidv4().replace(/-/g, "").slice(0, 6); - const ext = path.extname(file) - const baseName = path.basename(file, ext) - const updatedFileName = `${baseName}_${uuid}${ext}` - fileMap.set(updatedFileName, file) - return getFilePath(updatedFileName) - }) - return { filesList: fileMap, updatedFileNames } - -} - -const getURLExpiry = (access: string) => { - return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry -} - const checkLimitExceed = (files: Array): boolean => { return _.size(files) > maxFiles } diff --git a/api-service/src/v2/controllers/GenerateSignedURL/helper.ts b/api-service/src/v2/controllers/GenerateSignedURL/helper.ts new file mode 100644 index 00000000..c812c56a --- /dev/null +++ b/api-service/src/v2/controllers/GenerateSignedURL/helper.ts @@ -0,0 +1,64 @@ +import { config } from "../../configs/Config"; +import * as _ from "lodash"; +import { cloudProvider } from "../../services/CloudServices"; +import { URLAccess } from "../../types/SampleURLModel"; +import path from "path"; +import { v4 as uuidv4 } from "uuid"; + +export const generatePreSignedUrl = async (access: string, files: any, containerType: string) => { + const { filesList, updatedFileNames } = transformFileNames(files, access, containerType); + const urlExpiry: number = getURLExpiry(access); + const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)); + const signedUrlList = _.map(preSignedUrls, list => { + const fileNameWithUid = _.keys(list)[0]; + return { + filePath: getFilePath(fileNameWithUid, containerType), + fileName: filesList.get(fileNameWithUid), + preSignedUrl: _.values(list)[0] + }; + }); + return signedUrlList; +} + +const getFilePath = (file: string, containerType: string) => { + const datasetUploadPath = `${config.presigned_url_configs.service}/user_uploads/${file}`; + const connectorUploadPath = `${config.cloud_config.container_prefix}/${file}`; + const paths: Record = { + "dataset": datasetUploadPath, + "connector": connectorUploadPath + }; + return paths[containerType] || datasetUploadPath; +} + +const transformFileNames = (fileList: Array, access: string, containerType: string): Record => { + if (access === URLAccess.Read) { + return transformReadFiles(fileList, containerType); + } + return transformWriteFiles(fileList, containerType); +} + +const transformReadFiles = (fileNames: Array, containerType: string) => { + const fileMap = new Map(); + const updatedFileNames = _.map(fileNames, file => { + fileMap.set(file, file); + return getFilePath(file, containerType); + }); + return { filesList: fileMap, updatedFileNames }; +} + +const transformWriteFiles = (fileNames: Array, containerType: string) => { + const fileMap = new Map(); + const updatedFileNames = _.map(fileNames, file => { + const uuid = uuidv4().replace(/-/g, "").slice(0, 6); + const ext = path.extname(file); + const baseName = path.basename(file, ext); + const updatedFileName = `${baseName}_${uuid}${ext}`; + fileMap.set(updatedFileName, file); + return getFilePath(updatedFileName, containerType); + }); + return { filesList: fileMap, updatedFileNames }; +} + +const getURLExpiry = (access: string) => { + return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry; +} From 490f3575e61c2af9e56b17cd5a1638b5fd1f02d6 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 16:37:53 +0530 Subject: [PATCH 032/311] Sanketika-obsrv/issue-tracker: #OBS-150: removed config --- api-service/src/v2/configs/Config.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 60a048c1..c1958580 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -96,8 +96,7 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), - "path": process.env.command_service_path || "/system/v1/dataset/command", - "connector_registry_path": process.env.connector_registry_path || "/connector/v1/register", + "path": process.env.command_service_path || "/system/v1/dataset/command" }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", From 2ca6e08e4bb1eb047a98dfdc90bdecfea5438ac2 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 16:38:55 +0530 Subject: [PATCH 033/311] Sanketika-obsrv/issue-tracker: #OBS-150: added connector registry call in connector register connection --- api-service/src/v2/connections/commandServiceConnection.ts | 6 +++++- .../ConnectorRegistryStreamController.ts | 7 ++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api-service/src/v2/connections/commandServiceConnection.ts b/api-service/src/v2/connections/commandServiceConnection.ts index 94597474..63912747 100644 --- a/api-service/src/v2/connections/commandServiceConnection.ts +++ b/api-service/src/v2/connections/commandServiceConnection.ts @@ -18,4 +18,8 @@ export const executeCommand = async (id: string, command: string) => { } } return commandHttpService.post(commandPath, payload) -} \ No newline at end of file +} + +export const connectorRegistry = async (registryRequestBody: any) => { + return commandHttpService.post("/connector/v1/register", registryRequestBody) +} diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts index dc66d595..b4415727 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts @@ -2,19 +2,16 @@ import { Request, Response } from "express"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import _ from "lodash"; import logger from "../../logger"; -import { config } from "../../configs/Config"; import axios from "axios"; import httpStatus from "http-status"; import busboy from "busboy"; import { PassThrough } from "stream"; import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; +import { connectorRegistry } from "../../connections/commandServiceConnection"; export const apiId = "api.connector.stream.upload"; export const code = "FAILED_TO_REGISTER_CONNECTOR"; -const commandServiceHost = _.get(config, ["command_service_config", "host"]); -const commandServicePort = _.get(config, ["command_service_config", "port"]); -const registryUrl = _.get(config, ["command_service_config", "connector_registry_path"]) let resmsgid: string | any; const connectorRegistryStream = async (req: Request, res: Response) => { @@ -25,7 +22,7 @@ const connectorRegistryStream = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await axios.post(`${commandServiceHost}:${commandServicePort}${registryUrl}`, registryRequestBody); + const registryResponse = await connectorRegistry(registryRequestBody); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { From e5e52ae76684ec910fb946ab402ecd2a7e5ddbb7 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 16:42:22 +0530 Subject: [PATCH 034/311] Sanketika-obsrv/issue-tracker: #OBS-150: removed lodash map --- .../src/v2/controllers/GenerateSignedURL/helper.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api-service/src/v2/controllers/GenerateSignedURL/helper.ts b/api-service/src/v2/controllers/GenerateSignedURL/helper.ts index c812c56a..fc42e2e1 100644 --- a/api-service/src/v2/controllers/GenerateSignedURL/helper.ts +++ b/api-service/src/v2/controllers/GenerateSignedURL/helper.ts @@ -35,20 +35,20 @@ const transformFileNames = (fileList: Array, access: string, conta return transformReadFiles(fileList, containerType); } return transformWriteFiles(fileList, containerType); -} +}; const transformReadFiles = (fileNames: Array, containerType: string) => { const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { + const updatedFileNames = fileNames.map(file => { fileMap.set(file, file); return getFilePath(file, containerType); }); return { filesList: fileMap, updatedFileNames }; -} +}; const transformWriteFiles = (fileNames: Array, containerType: string) => { const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { + const updatedFileNames = fileNames.map(file => { const uuid = uuidv4().replace(/-/g, "").slice(0, 6); const ext = path.extname(file); const baseName = path.basename(file, ext); @@ -57,7 +57,7 @@ const transformWriteFiles = (fileNames: Array, containerType: stri return getFilePath(updatedFileName, containerType); }); return { filesList: fileMap, updatedFileNames }; -} +}; const getURLExpiry = (access: string) => { return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry; From dd4355a303aecb81df1bcef4a2a274532d0e9501 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 18:37:05 +0530 Subject: [PATCH 035/311] Sanketika-obsrv/issue-tracker: #OBS-150: changed route name and added connector register service in command api connection --- .../connections/commandServiceConnection.ts | 2 +- .../ConnectorRegisterController.ts} | 22 +++++++++---------- api-service/src/v2/routes/Router.ts | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) rename api-service/src/v2/controllers/{ConnectorRegistryStream/ConnectorRegistryStreamController.ts => ConnectorRegister/ConnectorRegisterController.ts} (88%) diff --git a/api-service/src/v2/connections/commandServiceConnection.ts b/api-service/src/v2/connections/commandServiceConnection.ts index 63912747..3f1582fa 100644 --- a/api-service/src/v2/connections/commandServiceConnection.ts +++ b/api-service/src/v2/connections/commandServiceConnection.ts @@ -20,6 +20,6 @@ export const executeCommand = async (id: string, command: string) => { return commandHttpService.post(commandPath, payload) } -export const connectorRegistry = async (registryRequestBody: any) => { +export const registerConnector = async (registryRequestBody: any) => { return commandHttpService.post("/connector/v1/register", registryRequestBody) } diff --git a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts b/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts similarity index 88% rename from api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts rename to api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts index b4415727..ca21f9d7 100644 --- a/api-service/src/v2/controllers/ConnectorRegistryStream/ConnectorRegistryStreamController.ts +++ b/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -7,14 +7,14 @@ import httpStatus from "http-status"; import busboy from "busboy"; import { PassThrough } from "stream"; import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; -import { connectorRegistry } from "../../connections/commandServiceConnection"; +import { registerConnector } from "../../connections/commandServiceConnection"; -export const apiId = "api.connector.stream.upload"; +export const apiId = "api.connector.register"; export const code = "FAILED_TO_REGISTER_CONNECTOR"; let resmsgid: string | any; -const connectorRegistryStream = async (req: Request, res: Response) => { +const connectorRegisterController = async (req: Request, res: Response) => { resmsgid = _.get(res, "resmsgid"); try { const uploadStreamResponse: any = await uploadStream(req); @@ -22,7 +22,7 @@ const connectorRegistryStream = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await connectorRegistry(registryRequestBody); + const registryResponse = await registerConnector(registryRequestBody); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { @@ -40,14 +40,14 @@ const connectorRegistryStream = async (req: Request, res: Response) => { const uploadStream = async (req: Request) => { return new Promise((resolve, reject) => { const filePromises: Promise[] = []; - const bb = busboy({ headers: req.headers }); + const busboyClient = busboy({ headers: req.headers }); const relative_path: any[] = []; let fileCount = 0; - bb.on("file", async (name: any, file: any, info: any) => { + busboyClient.on("file", async (name: any, file: any, info: any) => { if (fileCount > 0) { // If more than one file is detected, reject the request - bb.emit("error", reject({ + busboyClient.emit("error", reject({ code: "FAILED_TO_UPLOAD", message: "Uploading multiple files are not allowed", statusCode: 400, @@ -85,7 +85,7 @@ const uploadStream = async (req: Request) => { }; filePromises.push(processFile()); }); - bb.on("close", async () => { + busboyClient.on("close", async () => { try { await Promise.all(filePromises); resolve(relative_path); @@ -99,8 +99,8 @@ const uploadStream = async (req: Request) => { }); } }); - bb.on("error", reject); - req.pipe(bb); + busboyClient.on("error", reject); + req.pipe(busboyClient); }) } @@ -118,4 +118,4 @@ const extractFileNameFromPath = (filePath: string): string[] => { return filePath.match(regex) || []; }; -export default connectorRegistryStream; +export default connectorRegisterController; diff --git a/api-service/src/v2/routes/Router.ts b/api-service/src/v2/routes/Router.ts index e5e47105..de72d93c 100644 --- a/api-service/src/v2/routes/Router.ts +++ b/api-service/src/v2/routes/Router.ts @@ -20,7 +20,7 @@ import { eventValidation } from "../controllers/EventValidation/EventValidation" import GenerateSignedURL from "../controllers/GenerateSignedURL/GenerateSignedURL"; import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; import DatasetStatusTansition from "../controllers/DatasetStatusTransition/DatasetStatusTransition"; -import connectorRegistryStream from "../controllers/ConnectorRegistryStream/ConnectorRegistryStreamController"; +import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; import datasetHealth from "../controllers/DatasetHealth/DatasetHealth"; export const router = express.Router(); @@ -41,7 +41,7 @@ router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); -router.post("/connector/stream/upload", setDataToRequestObject("api.connector.stream.upload"), onRequest({ entity: Entity.Management }), connectorRegistryStream); +router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); //Wrapper Service From a9d0be318a18898fac729c5ee4812382c9520f0b Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 18:53:17 +0530 Subject: [PATCH 036/311] Sanketika-obsrv/issue-tracker: #OBS-150: changed route name and added connector register service in command api connection --- .../ConnectorRegister/ConnectorRegisterController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts index ca21f9d7..3a81f771 100644 --- a/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -18,11 +18,11 @@ const connectorRegisterController = async (req: Request, res: Response) => { resmsgid = _.get(res, "resmsgid"); try { const uploadStreamResponse: any = await uploadStream(req); - const registryRequestBody = { + const payload = { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await registerConnector(registryRequestBody); + const registryResponse = await registerConnector(payload); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { From bd485f9b852f882d45c84c5457788b08619dbde5 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 1 Jul 2024 19:01:26 +0530 Subject: [PATCH 037/311] Sanketika-obsrv/issue-tracker: #OBS-150: changed route name and added connector register service in command api connection --- api-service/src/v2/connections/commandServiceConnection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/connections/commandServiceConnection.ts b/api-service/src/v2/connections/commandServiceConnection.ts index 3f1582fa..541bded6 100644 --- a/api-service/src/v2/connections/commandServiceConnection.ts +++ b/api-service/src/v2/connections/commandServiceConnection.ts @@ -20,6 +20,6 @@ export const executeCommand = async (id: string, command: string) => { return commandHttpService.post(commandPath, payload) } -export const registerConnector = async (registryRequestBody: any) => { - return commandHttpService.post("/connector/v1/register", registryRequestBody) +export const registerConnector = async (requestBody: any) => { + return commandHttpService.post("/connector/v1/register", requestBody) } From 876afad28ef469cf8e954c2984739fb1caf82cab Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 7 Jul 2024 23:41:57 +0530 Subject: [PATCH 038/311] #OBS-I116: Dataset CRUD APIs test cases and fixes --- .../DatasetCreate/DatasetCreate.spec.ts | 73 +---- .../DatasetCreate/Fixtures.ts | 286 +++++++----------- .../DatasetList/DatasetList.spec.ts | 80 +---- .../DatasetManagement/DatasetList/Fixtures.ts | 192 ++---------- .../DatasetRead/DatasetRead.spec.ts | 110 ++----- .../DatasetManagement/DatasetRead/Fixtures.ts | 209 ++++--------- 6 files changed, 239 insertions(+), 711 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 09b71821..0aec1b47 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -9,9 +9,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import { sequelize } from "../../../connections/databaseConnection"; import _ from "lodash"; import { apiId } from "../../../controllers/DatasetCreate/DatasetCreate" -import { DatasourceDraft } from "../../../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { DatasetTransformations } from "../../../models/Transformation"; import { Dataset } from "../../../models/Dataset"; chai.use(spies); @@ -31,32 +28,16 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) + }) chai.spy.on(sequelize, "query", () => { return Promise.resolve([{ nextVal: 9 }]) }) - chai.spy.on(DatasourceDraft, "create", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: { id: "telemetry" } }) }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "data_schema": {"$schema": "https://json-schema.org/draft/2020-12/schema","type": "object", - "properties": { - "eid": {"type": "string"}, - "ets": {"type": "string"} - }, - "additionalProperties": true - },}) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve() - }) - chai .request(app) .post("/v2/datasets/create") @@ -79,6 +60,9 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) + }) chai .request(app) @@ -135,50 +119,5 @@ describe("DATASET CREATE API", () => { }); }); - it("Dataset creation failure: When timestamp key does not exist in the data schema", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(null) - }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([{ nextVal: 9 }]) - }) - chai.spy.on(DatasetDraft, "create", () => { - return Promise.resolve({ dataValues: { id: "telemetry" } }) - }) - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.DATASET_WITH_INVALID_TIMESTAMP) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Provided timestamp key not found in the data schema") - res.body.error.code.should.be.eq("DATASET_TIMESTAMP_NOT_FOUND") - done(); - }); - }); - - it("Dataset creation failure: Connection to the database failed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.reject({}) - }) - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.DATASET_WITH_DUPLICATE_DENORM_KEY) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to create dataset") - res.body.error.code.should.be.eq("DATASET_CREATION_FAILURE") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index b2dbb7f3..b3138c9f 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -11,7 +11,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "validation_config": { "validate": true, @@ -44,49 +44,23 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "trip-details" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] - }, - "tags": [] - } - }, - - VALID_DATASET_WITH_DEFAULT_TS: { - "id": "api.datasets.create", - "ver": "v1", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "dataset_id": "sb-ddd", - "type": "dataset", - "name": "sb-telemetry2", - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - "additionalProperties": true - }, - "dataset_config": { - "data_key": "", - "timestamp_key": "obsrv_meta.syncts" + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [] } @@ -101,7 +75,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -123,27 +97,19 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, - "transformations_config": [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ], + "transformations_config": [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], "tags": [] } }, @@ -157,7 +123,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -179,42 +145,19 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] - }, - "transformations_config": [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ], + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "transformations_config": [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, { "field_key": "ver", "transformation_function": { "type": "mask", "expr": "ver", "datatype": "string", "category": "pii" }, "mode": "Strict" }], "tags": [] } }, @@ -228,7 +171,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -250,9 +193,18 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, @@ -265,7 +217,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -287,9 +239,18 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, VALID_MORE_THAN_MINIMAL_DATASET: { @@ -301,7 +262,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -325,14 +286,24 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master-telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, VALID_MORE_THAN_MINIMAL_MASTER_DATASET: { @@ -344,7 +315,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -368,14 +339,24 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - }, + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } } }, VALID_MASTER_DATASET: { @@ -387,7 +368,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "validation_config": { "validate": true, @@ -420,13 +401,23 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [] } @@ -444,41 +435,6 @@ export const TestInputsForDatasetCreate = { } }, - DATASET_WITH_INVALID_TIMESTAMP: { - "id": "api.datasets.create", - "ver": "v1", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "dataset_id": "sb-ddd", - "type": "dataset", - "name": "sb-telemetry2", - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "dataset_config": { - "data_key": "", - "timestamp_key": "lastAccessed" - }, - "tags": [] - } - }, - DATASET_WITH_DUPLICATE_DENORM_KEY: { "id": "api.datasets.create", "ver": "v2", @@ -488,7 +444,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -510,11 +466,13 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" }, { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] } @@ -585,14 +543,7 @@ export const DATASET_CREATE_SUCCESS_FIXTURES = [ "httpStatus": httpStatus.OK, "status": "SUCCESS", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - { - "title": "Dataset creation success: Geenerating ingestion spec successfully using the data schema", - "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_DEFAULT_TS, - "httpStatus": httpStatus.OK, - "status": "SUCCESS", - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, + } ] export const DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES = [ @@ -602,12 +553,5 @@ export const DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES = [ "httpStatus": httpStatus.BAD_REQUEST, "status": "FAILED", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - { - "title": "Master Dataset creation failure: Dataset contains duplicate denorm out field", - "requestPayload": _.set(TestInputsForDatasetCreate.DATASET_WITH_DUPLICATE_DENORM_KEY, "request.type", "master-dataset"), - "httpStatus": httpStatus.BAD_REQUEST, - "status": "FAILED", - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" } ] \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts index 67344e5a..8b55ee75 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts @@ -5,12 +5,11 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetList/DatasetList"; +import { apiId } from "../../../controllers/DatasetList/DatasetList"; import { TestInputsForDatasetList } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; chai.use(spies); chai.should(); @@ -31,12 +30,6 @@ describe("DATASET LIST API", () => { chai.spy.on(DatasetDraft, "findAll", () => { return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) chai .request(app) .post("/v2/datasets/list") @@ -50,7 +43,7 @@ describe("DATASET LIST API", () => { res.body.result.count.should.be.eq(2) res.body.params.msgid.should.be.eq(msgid) const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }, { ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }]) + const expectedResult = JSON.stringify(TestInputsForDatasetList.VALID_RESPONSE) result.should.be.eq(expectedResult) done(); }); @@ -60,11 +53,8 @@ describe("DATASET LIST API", () => { chai.spy.on(DatasetDraft, "findAll", () => { return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([]) }) chai .request(app) @@ -79,7 +69,7 @@ describe("DATASET LIST API", () => { res.body.result.count.should.be.eq(1) res.body.params.msgid.should.be.eq(msgid) const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }]) + const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA }]) result.should.be.eq(expectedResult) done(); }); @@ -89,11 +79,8 @@ describe("DATASET LIST API", () => { chai.spy.on(Dataset, "findAll", () => { return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) + chai.spy.on(DatasetDraft, "findAll", () => { + return Promise.resolve([]) }) chai .request(app) @@ -108,39 +95,7 @@ describe("DATASET LIST API", () => { res.body.result.count.should.be.eq(1) res.body.params.msgid.should.be.eq(msgid) const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }]) - result.should.be.eq(expectedResult) - done(); - }); - }); - - it("Dataset list success: When sortBy is provided in request payload", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITH_SORTBY) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.count.should.be.eq(2) - res.body.params.msgid.should.be.eq(msgid) - const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }, { ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }]) + const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA}]) result.should.be.eq(expectedResult) done(); }); @@ -162,23 +117,4 @@ describe("DATASET LIST API", () => { }); }); - - it("Dataset list failure: Connection to the database failed", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.reject() - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITHOUT_FILTERS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to list dataset") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts index a905b1c0..31a5ba1c 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts @@ -1,181 +1,53 @@ export const TestInputsForDatasetList = { VALID_DRAFT_DATASET_SCHEMA: { "dataset_id": "telemetry", - "id": "telemetry.1", "name": "telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "msgid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] + "type": "events", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" ] }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, "tags": [ "tag1", "tag2" ], "status": "Draft", "version": 1, - "client_state": {}, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "2024-04-15 07:51:49.49", - "update_date": "", - "published_date": "" + "api_version": "v2" }, VALID_LIVE_DATASET_SCHEMA: { "dataset_id": "sb-telemetry", - "id": "sb-telemetry", "name": "sb-telemetry", - "type": "master-dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "msgid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] + "type": "master", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" ] }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, "tags": [ "tag1", "tag2" ], "status": "Live", "data_version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "2024-04-16 07:51:49.49", - "update_date": "", - "published_date": "" - }, - TRANSFORMATIONS_DRAFT_SCHEMA: { - "dataset_id": "telemetry.1", - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - }, - TRANSFORMATIONS_LIVE_SCHEMA: { - "dataset_id": "sb-telemetry", - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - } + "api_version": "v2" }, REQUEST_WITHOUT_FILTERS: { @@ -206,18 +78,7 @@ export const TestInputsForDatasetList = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "filters": { status: "Live", type: "master-dataset" } - } - }, - REQUEST_WITH_SORTBY: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "sortBy": [{ "field": "created_date", "order": "asc" }] + "filters": { status: "Live", type: "master" } } }, INVALID_REQUEST: { @@ -230,5 +91,6 @@ export const TestInputsForDatasetList = { "request": { "filters": { status: ["Ready"] } } - } + }, + VALID_RESPONSE: [{"dataset_id":"sb-telemetry","name":"sb-telemetry","type":"master","dataset_config":{"indexing_config":{"olap_store_enabled":false,"lakehouse_enabled":true,"cache_enabled":true},"keys_config":{"data_key":"ets"},"file_upload_path":["telemetry.json"]},"tags":["tag1","tag2"],"status":"Live","data_version":1,"api_version":"v2"},{"dataset_id":"telemetry","name":"telemetry","type":"events","dataset_config":{"indexing_config":{"olap_store_enabled":false,"lakehouse_enabled":true,"cache_enabled":false},"keys_config":{"timestamp_key":"ets"},"file_upload_path":["telemetry.json"]},"tags":["tag1","tag2"],"status":"Draft","version":1,"api_version":"v2"}] } \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index dce1b54c..97ec0282 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -8,13 +8,10 @@ import _ from "lodash"; import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; import { TestInputsForDatasetRead } from "./Fixtures"; import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; -import { Datasource } from "../../../models/Datasource"; import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; -import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; -import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { ConnectorInstances } from "../../../models/ConnectorInstances"; chai.use(spies); chai.should(); @@ -27,8 +24,8 @@ describe("DATASET READ API", () => { }); it("Dataset read success: When minimal fields requested", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ 'name': 'sb-telemetry', 'data_version': 1 }]) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ 'name': 'sb-telemetry', 'data_version': 1 }) }) chai .request(app) @@ -41,41 +38,35 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq('sb-telemetry') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ 'name': 'sb-telemetry', 'version': 1 })) + result.should.be.eq(JSON.stringify({ name: 'sb-telemetry', data_version: 1 })) done(); }); }); it("Dataset read success: Fetch all dataset fields when fields param is empty", (done) => { - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DRAFT_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.DRAFT_SCHEMA) }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.type.should.be.eq('dataset') + res.body.result.type.should.be.eq('event') res.body.result.status.should.be.eq('Draft') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA, "transformations_config": [] })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA })) done(); }); }); it("Dataset read success: Fetch live dataset when status param is empty", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) chai .request(app) @@ -88,12 +79,12 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.status.should.be.eq('Live') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ..._.omit({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "transformations_config": TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA }, ["data_version"]), version: 1 })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) done(); }); }); - it("Dataset read success: Creating draft on mode=edit if no draft found", (done) => { + it("Dataset read success: Creating draft on mode=edit if no draft found in v2", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() }) @@ -103,30 +94,15 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetTransformations, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DATASOURCE_SCHEMA]) - }) - chai.spy.on(DatasetSourceConfig, "findAll", () => { + chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve([]) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfigDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft&mode=edit") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -135,41 +111,26 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq('sb-telemetry') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA, "transformations_config": [] })) + result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); }); }); - it("Dataset read success: Updating dataset status to draft on mode=edit if dataset status is Live", (done) => { + it("Dataset read success: Creating draft on mode=edit if no draft found in v1", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "sb-telemetry", name: "sb-telemetry", status: "Live", data_schema: {} }) + return Promise.resolve() }) chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "sb-telemetry", name: "sb-telemetry", status: "Live", data_version: 2, data_schema: {} }) + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) }) chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DATASOURCE_SCHEMA]) + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) }) chai.spy.on(DatasetSourceConfig, "findAll", () => { return Promise.resolve([]) }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfigDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) + chai.spy.on(DatasetDraft, "create", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) chai .request(app) @@ -181,8 +142,8 @@ describe("DATASET READ API", () => { res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.result.name.should.be.eq('sb-telemetry') - const result = JSON.stringify(_.omit(res.body.result, "version_key")) - result.should.be.eq(JSON.stringify({"dataset_id":"sb-telemetry","name":"sb-telemetry","data_schema":{},"version":2,"status":"Draft","api_version":"v2","transformations_config":[]})) + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); }); }); @@ -209,8 +170,8 @@ describe("DATASET READ API", () => { }); it("Dataset read failure: When the dataset of requested dataset_id not found", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([]) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) }) chai .request(app) @@ -244,7 +205,7 @@ describe("DATASET READ API", () => { it("Dataset read failure: When specified field of draft dataset cannot be found", (done) => { chai .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=data&status=Draft") + .get("/v2/datasets/read/sb-telemetry?fields=data&mode=edit") .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") @@ -256,21 +217,4 @@ describe("DATASET READ API", () => { }); }); - it("Dataset read failure: Connection to the database failed", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.reject() - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry") - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to read dataset") - res.body.error.code.should.be.eq("DATASET_READ_FAILURE") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts index 93c118e8..61760c6e 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -1,153 +1,56 @@ export const TestInputsForDatasetRead = { DRAFT_SCHEMA: { "dataset_id": "sb-telemetry", - "id": "sb-telemetry", "name": "sb-telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, + "type": "event", + "status": "Draft", "tags": [ "tag1", "tag2" ], - "status": "Draft", "version": 1, - "client_state": {}, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "", - "update_date": "", - "published_date": "" + "api_version": "v2", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } }, LIVE_SCHEMA: { - "dataset_id": "sb-telemetry", - "id": "sb-telemetry", "name": "sb-telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, + "type": "event", + "status": "Live", "tags": [ "tag1", "tag2" ], - "status": "Live", "data_version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "", - "update_date": "", - "published_date": "" + "api_version": "v2", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } }, - TRANSFORMATIONS_SCHEMA: [ + TRANSFORMATIONS_SCHEMA:[{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + TRANSFORMATIONS_SCHEMA_V1: [ { "field_key": "eid", "transformation_function": { @@ -164,27 +67,27 @@ export const TestInputsForDatasetRead = { } } ], - DATASOURCE_SCHEMA:{ - "id": "sb-telemetry_sb-telemetry", - "datasource": "sb-telemetry", - "dataset_id": "sb-telemetry", - "ingestion_spec": {"type":"kafka","spec":{"dataSchema":{"dataSource":"dataset-conf_day","dimensionsSpec":{"dimensions":[{"type":"string","name":"a"},{"type":"string","name":"obsrv.meta.source.connector"},{"type":"string","name":"obsrv.meta.source.id"}]},"timestampSpec":{"column":"obsrv_meta.syncts","format":"auto"},"metricsSpec":[],"granularitySpec":{"type":"uniform","segmentGranularity":"DAY","queryGranularity":"none","rollup":false}},"tuningConfig":{"type":"kafka","maxBytesInMemory":134217728,"maxRowsPerSegment":5000000,"logParseExceptions":true},"ioConfig":{"type":"kafka","consumerProperties":{"bootstrap.servers":"localhost:9092"},"taskCount":1,"replicas":1,"taskDuration":"PT1H","useEarliestOffset":true,"completionTimeout":"PT1H","inputFormat":{"type":"json","flattenSpec":{"useFieldDiscovery":true,"fields":[{"type":"path","expr":"$.['a']","name":"a"},{"type":"path","expr":"$.obsrv_meta.['syncts']","name":"obsrv_meta.syncts"},{"type":"path","expr":"$.obsrv_meta.source.['connector']","name":"obsrv.meta.source.connector"},{"type":"path","expr":"$.obsrv_meta.source.['connectorInstance']","name":"obsrv.meta.source.id"},{"expr":"$.obsrv_meta.syncts","name":"obsrv_meta.syncts","type":"path"}]}},"appendToExisting":false}}}, - "datasource_ref": "sb-telemetry_DAY", - "retention_period": { - "enabled": "false" - }, - "archival_policy": { - "enabled": "false" - }, - "purge_policy": { - "enabled": "false" - }, - "backup_config": { - "enabled": "false" - }, - "status": "Live", - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "published_date": "2023-07-03 00:00:00" + DATASOURCE_SCHEMA: { + "id": "sb-telemetry_sb-telemetry", + "datasource": "sb-telemetry", + "dataset_id": "sb-telemetry", + "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "dataset-conf_day", "dimensionsSpec": { "dimensions": [{ "type": "string", "name": "a" }, { "type": "string", "name": "obsrv.meta.source.connector" }, { "type": "string", "name": "obsrv.meta.source.id" }] }, "timestampSpec": { "column": "obsrv_meta.syncts", "format": "auto" }, "metricsSpec": [], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "none", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 5000000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "consumerProperties": { "bootstrap.servers": "localhost:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [{ "type": "path", "expr": "$.['a']", "name": "a" }, { "type": "path", "expr": "$.obsrv_meta.['syncts']", "name": "obsrv_meta.syncts" }, { "type": "path", "expr": "$.obsrv_meta.source.['connector']", "name": "obsrv.meta.source.connector" }, { "type": "path", "expr": "$.obsrv_meta.source.['connectorInstance']", "name": "obsrv.meta.source.id" }, { "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts", "type": "path" }] } }, "appendToExisting": false } } }, + "datasource_ref": "sb-telemetry_DAY", + "retention_period": { + "enabled": "false" + }, + "archival_policy": { + "enabled": "false" + }, + "purge_policy": { + "enabled": "false" + }, + "backup_config": { + "enabled": "false" + }, + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "published_date": "2023-07-03 00:00:00" } } \ No newline at end of file From 32d0bbe73e93683fbfa76cdd060df1af4bd54d7a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 10 Jul 2024 23:16:48 +0530 Subject: [PATCH 039/311] #OBS-I116: Dataset update API Dedupe and denorm test cases fixes --- .../DatasetStatusTransition.ts | 5 +- .../DatasetUpdate/DatasetDedup.spec.ts | 6 +- .../DatasetUpdate/DatasetDenorm.spec.ts | 83 ++----------------- .../DatasetUpdate/Fixtures.ts | 19 +++-- 4 files changed, 22 insertions(+), 91 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 8512af8e..18fe3de4 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -22,6 +22,7 @@ import { druidHttpService } from "../../connections/druidConnection"; import { query, sequelize } from "../../connections/databaseConnection"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +export const apiId = "api.datasets.status-transition"; const transitionFailed = "DATASET_STATUS_TRANSITION_FAILURE" const invalidRequest = "DATASET_STATUS_TRANSITION_INVALID_INPUT" const datasetNotFound = "DATASET_NOT_FOUND" @@ -79,10 +80,10 @@ const validateDataset = (req: Request, res: Response, dataset: any, action: stri if(!_.includes(allowedTransitions[action], dataset.status)) { const code = `DATASET_${_.toUpper(action)}_FAILURE` - logger.error({ code, headers, message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}` }) + logger.error({ code, headers, message: `for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}` }) ResponseHandler.errorResponse({ code: datasetNotFound, - message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, + message: `$ for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, statusCode: 404, errCode: "NOT_FOUND" } as ErrorObject, req, res); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index 8d27057e..aa3ae5da 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -22,7 +22,7 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Success: Dataset dedupe configs updated with dedup key if duplicates need to be dropped", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Success: Dataset dedupe configs updated with default values if duplicates need to be dropped", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -57,7 +57,7 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, dedup_config: { drop_duplicates: false } } }) + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, dedup_config: { drop_duplicates: false, dedup_key: "mid" } } }) .end((err, res) => { console.log(res.body.result) res.should.have.status(httpStatus.OK); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index 268b83c8..aaddfd4a 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -22,7 +22,7 @@ describe("DATASET DENORM UPDATE", () => { it("Success: Dataset denorms successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, denorm_config: { denorm_field: [] } + id: "telemetry", status: "Draft", type:"event", version_key: validVersionKey, api_version:"v2", denorm_config: { denorm_field: [] } }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET DENORM UPDATE", () => { it("Success: Dataset denorms successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, denorm_config: { denorm_fields: [{ denorm_out_field: "userdata" }] } + id: "telemetry", status: "Draft", type:"event", version_key: validVersionKey, api_version:"v2", denorm_config: { denorm_fields: [{ denorm_out_field: "userdata" }] } }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -74,10 +74,11 @@ describe("DATASET DENORM UPDATE", () => { it("Success: When payload contains same denorms to be removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", version_key: validVersionKey, type:"dataset", status: "Draft", denorm_config: { + id: "telemetry", version_key: validVersionKey, type:"dataset", api_version:"v2", status: "Draft", denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }] } }) @@ -102,78 +103,4 @@ describe("DATASET DENORM UPDATE", () => { }); }); - - it("Failure: Dataset contains duplicate denorm field", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ status: "Draft", version_key: validVersionKey }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DUPLICATE_DENORM_KEY) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - expect(res.body.error.message).to.match(/^Dataset contains duplicate denorm out keys(.+)$/) - res.body.error.code.should.be.eq("DATASET_DUPLICATE_DENORM_KEY") - done(); - }); - }); - - it("Failure: When denorm fields provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - }] - } - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm fields already exist") - res.body.error.code.should.be.eq("DATASET_DENORM_EXISTS") - done(); - }); - }); - - it("Failure: When denorm fields provided to delete does not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "id" - }] - } - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm fields do not exist to remove") - res.body.error.code.should.be.eq("DATASET_DENORM_DO_NOT_EXIST") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 4dd1455e..b5652fa4 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -56,11 +56,12 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master" }, - "action": "add" + "action": "upsert" } ] } @@ -74,7 +75,7 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, @@ -407,16 +408,18 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" }, { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" } From 0a39770cc7f32061aaa26418799ddc6527fb61c3 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 12 Jul 2024 16:22:00 +0530 Subject: [PATCH 040/311] #OBS-I116: Dataset Create api test case fixes --- .../DatasetManagement/DatasetCreate/DatasetCreate.spec.ts | 4 ++-- .../tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 0aec1b47..b1cf2db2 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -74,7 +74,7 @@ describe("DATASET CREATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq(fixture.status) res.body.params.msgid.should.be.eq(fixture.msgid) - res.body.error.message.should.be.eq("Duplicate denorm key found") + res.body.error.message.should.be.eq("Duplicate denorm output fields found.") res.body.error.code.should.be.eq("DATASET_DUPLICATE_DENORM_KEY") done(); }); @@ -113,7 +113,7 @@ describe("DATASET CREATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset already exists") + res.body.error.message.should.be.eq("Dataset Already exists with id:sb-ddd") res.body.error.code.should.be.eq("DATASET_EXISTS") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 97ec0282..6accc269 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -181,7 +181,7 @@ describe("DATASET READ API", () => { res.body.should.be.a("object") res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Dataset with the given dataset_id not found") + res.body.error.message.should.be.eq("Dataset with the given dataset_id:sb-telemetry not found") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); From 0cd3eb3ec3528e33e2c3159c9db699898385dc68 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 12 Jul 2024 16:37:41 +0530 Subject: [PATCH 041/311] #OBS-I116: Dataset update extraction config api test case fixes --- .../DatasetUpdate/DatasetExtraction.spec.ts | 18 ++++- .../DatasetUpdate/DatasetTags.spec.ts | 74 +------------------ .../DatasetUpdate/Fixtures.ts | 64 ++++------------ 3 files changed, 31 insertions(+), 125 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 4d7c8131..923f2c5e 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -22,7 +22,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Success: Dataset extraction configs updated if it is a batch event", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, api_version: "v2", type: "event" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Success: Dataset extraction configs updated with default values if it is not batch event", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, api_version: "v2", type: "event" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -57,7 +57,19 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, extraction_config: { "is_batch_event": false } } }) + .send({ + ...requestStructure, request: { + dataset_id: "telemetry", version_key: validVersionKey, + "extraction_config": { + "is_batch_event": false, + "extraction_key": "events", + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "id" + } + } + } + }) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts index 90bb6c67..07471544 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -22,7 +22,7 @@ describe("DATASET TAGS UPDATE", () => { it("Success: Dataset tags successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, denorm_config: { denorm_fields: [] } + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, denorm_config: { denorm_fields: [] }, api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET TAGS UPDATE", () => { it("Success: Dataset tags successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"] + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -71,74 +71,4 @@ describe("DATASET TAGS UPDATE", () => { }); }); - it("Success: When payload contains same tags to be added or removed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"] - }) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_TAGS_ADD) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Failure: When tags provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag3", "tag1"] - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset tags already exist") - res.body.error.code.should.be.eq("DATASET_TAGS_EXISTS") - done(); - }); - }); - - it("Failure: When tags provided to delete does not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag5"] - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset tags do not exist to remove") - res.body.error.code.should.be.eq("DATASET_TAGS_DO_NOT_EXIST") - done(); - }); - }); - }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index b5652fa4..4e7e9b10 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -25,11 +25,8 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": [ - "tag1", - "tag2" - ], - "action": "add" + "values": "tag1", + "action": "upsert" }] } }, @@ -40,10 +37,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": [ - "tag1", - "tag2" - ], + "values": "tag1", "action": "remove" }] } @@ -98,7 +92,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }] } }, @@ -287,7 +281,7 @@ export const TestInputsForDatasetUpdate = { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" }, { "values": { @@ -306,7 +300,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }, { "values": { @@ -334,18 +328,12 @@ export const TestInputsForDatasetUpdate = { }, "tags": [ { - "values": [ - "tag1", - "tag2" - ], + "values": "tag1", "action": "remove" }, { - "values": [ - "tag3", - "tag4" - ], - "action": "add" + "values": "tag3", + "action": "upsert" } ] } @@ -362,44 +350,20 @@ export const TestInputsForDatasetUpdate = { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" }, { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" } ] } } }, - - DATASET_UPDATE_WITH_SAME_TAGS_ADD: { - ...requestStructure, request: { - "dataset_id": "telemetry", - "version_key": validVersionKey, - "name": "sb-telemetry", - "tags": [ - { - "values": [ - "tag1", - "tag1" - ], - "action": "remove" - }, - { - "values": [ - "tag4", - "tag4" - ], - "action": "add" - } - ] - } - }, - + DATASET_UPDATE_WITH_SAME_DENORM_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", @@ -441,7 +405,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }, { "values": { @@ -450,7 +414,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }, { "values": { From 80ac93d3dca669957986d86153dd7d1453b4718d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sat, 20 Jul 2024 19:42:25 +0530 Subject: [PATCH 042/311] #OBS-I116: Dataset update api test cases --- .../DatasetUpdate/DatasetDenorm.spec.ts | 2 +- .../DatasetTransformation.spec.ts | 152 +---------------- .../DatasetUpdate/DatasetUpdate.spec.ts | 128 ++------------ .../DatasetUpdate/DatasetValidation.spec.ts | 6 +- .../DatasetUpdate/Fixtures.ts | 156 +++++------------- 5 files changed, 57 insertions(+), 387 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index aaddfd4a..c0f21815 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -1,5 +1,5 @@ import app from "../../../../app"; -import chai, { expect } from "chai"; +import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index 14ae7c5a..6d38059c 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -7,7 +7,6 @@ import { describe, it } from 'mocha'; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" chai.use(spies); @@ -23,15 +22,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { it("Success: Dataset transformations successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", "transformations_config":[] }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) @@ -55,15 +48,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { it("Success: Dataset transformations successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", "transformations_config":[{ "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }] }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) @@ -84,53 +71,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { }); }); - it("Success: Dataset transformations successfully updated", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - it("Success: When payload contains same transformation field_key to be added, updated or removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) @@ -152,93 +95,4 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { }); }); - it("Failure: When transformation fields provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations already exists") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_EXIST") - done(); - }); - }); - - it("Failure: When transformation fields provided to update do not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset" , version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key7" }, { field_key: "key2" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations do not exist to update") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_DO_NOT_EXIST") - done(); - }); - }); - - it("Failure: When transformation fields provided to remove do not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key7" }, { field_key: "key3" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations do not exist to remove") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_DO_NOT_EXIST") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index d4d2f921..8ec55ff6 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -8,10 +8,9 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { apiId, errorCode, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { apiId, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" import { DatasourceDraft } from "../../../models/DatasourceDraft"; -import { Dataset } from "../../../models/Dataset"; -import { DatasetTransformations } from "../../../models/Transformation"; + chai.use(spies); chai.should(); @@ -25,7 +24,7 @@ describe("DATASET UPDATE API", () => { it("Dataset updation success: When minimal request payload provided", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) @@ -50,44 +49,14 @@ describe("DATASET UPDATE API", () => { it("Dataset updation success: When full request payload provided", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid" }] - } - }) - }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "properties": { - "eid": { "type": "string" }, - "ets": { "type": "string" } - }, - "additionalProperties": true - }, + }, api_version: "v2" }) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) @@ -141,7 +110,7 @@ describe("DATASET UPDATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset does not exists to update") + res.body.error.message.should.be.eq("Dataset does not exists with id:telemetry") res.body.error.code.should.be.eq("DATASET_NOT_EXISTS") done(); }); @@ -149,7 +118,7 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: When dataset to update is outdated", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: "1813444815918", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", type:"event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) }) chai @@ -170,7 +139,7 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: Dataset to update is not in draft state", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ status: "Live" }) + return Promise.resolve({ id: "telemetry", type:"event", status: "Live", version_key: "1713444815918", api_version: "v2" }) }) chai @@ -203,8 +172,7 @@ describe("DATASET UPDATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to update dataset") - res.body.error.code.should.be.eq(errorCode) + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR") done(); }); }); @@ -217,7 +185,7 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset name updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) @@ -267,39 +235,7 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset data schema updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DATA_SCHEMA_VALID) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: Ingestion spec updateded successfully", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetTransformationsDraft, "findAll", () => { @@ -328,46 +264,6 @@ describe("DATASET UPDATE API", () => { }); }); - it("Failure: When timestamp key does not exist in the data schema", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" - }) - }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "properties": { - "eid": { "type": "string" }, - "ets": { "type": "string" } - }, - "additionalProperties": true - }, - }) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_WITH_INVALID_TIMESTAMP) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Provided timestamp key not found in the data schema") - res.body.error.code.should.be.eq("DATASET_TIMESTAMP_NOT_FOUND") - done(); - }); - }); - it("Failure: Failed to update data schema", (done) => { chai .request(app) @@ -395,7 +291,7 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset config updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index a56444ae..1fc1aaed 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -22,7 +22,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Success: Dataset validation configs updated when validation is true", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Success: Dataset validation configs updated with default values when validation is false", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -57,7 +57,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, validation_config: { "validate": false } } }) + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, validation_config: { "validate": false, "mode": "Strict" } } }) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 4e7e9b10..eb2c683b 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -25,7 +25,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": "tag1", + "value": "tag1", "action": "upsert" }] } @@ -37,7 +37,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": "tag1", + "value": "tag1", "action": "remove" }] } @@ -84,16 +84,7 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" - }] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }], } }, @@ -193,8 +184,18 @@ export const TestInputsForDatasetUpdate = { "dataset_id": "telemetry", "version_key": validVersionKey, "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets", + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] } } }, @@ -203,16 +204,7 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }], } }, @@ -277,62 +269,45 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master" }, "action": "upsert" }, { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" } ] }, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "key2", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "remove" }], + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" + "keys_config": { + "timestamp_key": "ets", + "data_key": "ets" }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - } - ], - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [ { - "values": "tag1", + "value": "tag1", "action": "remove" }, { - "values": "tag3", + "value": "tag3", "action": "upsert" } ] @@ -363,7 +338,7 @@ export const TestInputsForDatasetUpdate = { } } }, - + DATASET_UPDATE_WITH_SAME_DENORM_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", @@ -397,62 +372,7 @@ export const TestInputsForDatasetUpdate = { "dataset_id": "telemetry", "version_key": validVersionKey, "name": "sb-telemetry", - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" - }, - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" - }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - } - ] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }] } } } From 8d284ece247751a88c562d9ea278b8f1b263f0f8 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sat, 20 Jul 2024 20:02:45 +0530 Subject: [PATCH 043/311] #OBS-I116: fix: linting fixes --- .../v2/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 22 +++---- .../DatasetUpdate/DatasetUpdate.ts | 4 +- .../GenerateDataSchema/GenerateDataSchema.ts | 22 +++---- .../exceptions/SchemaGenerationException.ts | 2 +- api-service/src/v2/middlewares/errors.ts | 6 +- api-service/src/v2/services/CipherService.ts | 10 +-- api-service/src/v2/services/DatasetService.ts | 12 ++-- .../SchemaGenerateService/ConfigSuggester.ts | 4 +- .../DataSchemaService.ts | 66 +++++++++---------- .../SchemaGenerateService/SchemaAnalyser.ts | 40 +++++------ .../SchemaArrayValidator.ts | 14 ++-- .../SchemaGeneratorUtils.ts | 6 +- .../SchemaGenerateService/SchemaHandler.ts | 42 ++++++------ .../SuggestionTemplate.ts | 4 +- .../SchemaGenerateService/Template.ts | 2 +- api-service/src/v2/services/TableGenerator.ts | 46 ++++++------- api-service/src/v2/types/ConfigModels.ts | 2 +- 18 files changed, 154 insertions(+), 152 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index c82008fc..9cd64d8f 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -16,7 +16,7 @@ const validateRequest = (req: Request) => { const { dataset_id } = req.params; const fields = req.query.fields; - if(fields && typeof fields !== 'string') { + if(fields && typeof fields !== "string") { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); } const fieldValues = fields ? _.split(fields, ",") : []; diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 948a7772..dac2daeb 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -102,7 +102,7 @@ const readyForPublish = async (dataset: Record) => { statusCode: 400 } } - _.set(draftDataset, 'status', DatasetStatus.ReadyToPublish) + _.set(draftDataset, "status", DatasetStatus.ReadyToPublish) await datasetService.updateDraftDataset(draftDataset) } @@ -129,7 +129,7 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) // 1. Check if there are denorm fields and dependent master datasets are published const denormConfig = _.get(draftDataset, "denorm_config") if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const datasetIds = _.map(denormConfig.denorm_fields, 'dataset_id') + const datasetIds = _.map(denormConfig.denorm_fields, "dataset_id") if(_.includes(datasetIds, draftDataset.id)) { throw { code: "SELF_REFERENCING_MASTER_DATA", @@ -141,22 +141,22 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) - let datasetStatus : Record = { + const datasetStatus : Record = { dataset_id: denormField.dataset_id, exists: (md) ? true : false, isLive: (md) ? md.status === "Live" : false, status: md.status } if(md.api_version === "v2") - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); else - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); return datasetStatus; }) const invalidMasters = _.filter(masterDatasetsStatus, {isLive: false}) if(_.size(invalidMasters) > 0) { - const invalidIds = _.map(invalidMasters, 'dataset_id') + const invalidIds = _.map(invalidMasters, "dataset_id") throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", message: `The datasets with id:${invalidIds} are not in published status`, @@ -169,13 +169,13 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) draftDataset["denorm_config"] = { redis_db_host: defaultDatasetConfig.denorm_config.redis_db_host, redis_db_port: defaultDatasetConfig.denorm_config.redis_db_port, - denorm_fields: _.map(masterDatasetsStatus, 'denorm_field') + denorm_fields: _.map(masterDatasetsStatus, "denorm_field") } } } const updateMasterDataConfig = async (draftDataset: Record) => { - if(draftDataset.type === 'master') { + if(draftDataset.type === "master") { if(draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() if(_.isEmpty(results)) { @@ -187,7 +187,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } const nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - _.set(draftDataset, 'dataset_config.cache_config.redis_db', nextRedisDB) + _.set(draftDataset, "dataset_config.cache_config.redis_db", nextRedisDB) } } } @@ -208,13 +208,13 @@ const canRetireIfMasterDataset = async (dataset: Record) => { const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) const extractDenormFields = _.map(allDatasets, function(depDataset) { - return {dataset_id: _.get(depDataset, 'id'), status: _.get(depDataset, 'status'), denorm_datasets: _.map(_.get(depDataset, 'denorm_config.denorm_fields'), 'dataset_id')} + return {dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id")} }) const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) if (_.size(deps) > 0) { const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` - throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, 'denorm_datasets')})) + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, "denorm_datasets")})) } } } diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index 1a8d68a5..e5384354 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -9,6 +9,7 @@ import { datasetService } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; import { obsrvError } from "../../types/ObsrvError"; +import logger from "../../logger"; export const apiId = "api.datasets.update"; export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" @@ -25,6 +26,7 @@ const validateRequest = async (req: Request) => { const datasetBody = req.body.request const { dataset_id, version_key, ...rest } = datasetBody if (_.isEmpty(rest)) { + logger.error({ apiId, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) } @@ -63,7 +65,7 @@ const datasetUpdate = async (req: Request, res: Response) => { const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { - let dataset: Record = { + const dataset: Record = { version_key: Date.now().toString(), name: datasetReq.name || _.get(datasetModel, ["name"]), id: _.get(datasetModel, ["id"]) diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts index 309dfe39..3d16f5b2 100644 --- a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -41,7 +41,7 @@ const schemaGenerate = (sample: Map[], config: Record) const schemaInference = new SchemaInference(); const schemaArrayValidator = new SchemaArrayValidator(); if (isJsonSchema) { - let result = process(sample, dataset) + const result = process(sample, dataset) result.schema = removeNonIndexColumns(result.schema) result.schema = removeFormats(result.schema) return result @@ -50,7 +50,7 @@ const schemaGenerate = (sample: Map[], config: Record) schema = schemaArrayValidator.validate(schema) const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) rollupInfo = schemaCardinalityAnalyser.analyse() - let result = process(schema, dataset) + const result = process(schema, dataset) result.schema = removeNonIndexColumns(result.schema) result.schema = removeFormats(result.schema) return result @@ -81,14 +81,14 @@ const checkJsonSchema = (sample: Map): boolean => { const removeNonIndexColumns = (schema: any) => { if (schema.properties) { Object.entries(schema.properties).map(([key, property]: any) => { - _.unset(schema, 'required'); + _.unset(schema, "required"); removeNonIndexColumns(property) }); } else if (schema.items) { removeNonIndexColumns(schema.items) } if (Array.isArray(schema.required) && schema.required.length === 0) { - _.unset(schema, 'required'); + _.unset(schema, "required"); } return schema } @@ -97,17 +97,17 @@ const removeFormats = (schema: any) => { if (schema.properties) { Object.entries(schema.properties).map(([key, property]: any) => { // Removing format to avoid schema validation issues - const isDateTypeField = ['date-time', 'date', 'epoch'].includes((property as any).format); - if (isDateTypeField && _.get(property, 'data_type') === 'string') { - _.set(property, 'data_type', _.get(property, 'format')); - } else if (isDateTypeField && _.get(property, 'data_type') === 'integer') { - _.set(property, 'data_type', 'epoch'); + const isDateTypeField = ["date-time", "date", "epoch"].includes((property as any).format); + if (isDateTypeField && _.get(property, "data_type") === "string") { + _.set(property, "data_type", _.get(property, "format")); + } else if (isDateTypeField && _.get(property, "data_type") === "integer") { + _.set(property, "data_type", "epoch"); } - _.unset(property, 'format'); + _.unset(property, "format"); removeFormats(property) }); } else if (schema.items) { - _.unset(schema.items, 'format'); + _.unset(schema.items, "format"); removeFormats(schema.items) } return schema diff --git a/api-service/src/v2/exceptions/SchemaGenerationException.ts b/api-service/src/v2/exceptions/SchemaGenerationException.ts index 44805da4..fc545e81 100644 --- a/api-service/src/v2/exceptions/SchemaGenerationException.ts +++ b/api-service/src/v2/exceptions/SchemaGenerationException.ts @@ -2,7 +2,7 @@ export class SchemaGenerationException extends Error { statusCode: number; constructor(message: string, code: number) { super(message); - this.name = 'SchemaGenerationException'; + this.name = "SchemaGenerationException"; this.statusCode = code; } } \ No newline at end of file diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index 568aaa67..2ca96caf 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -4,15 +4,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; -export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { +export const errorHandler = (err: Error, req: Request, res: Response) => { logger.error({ path: req.url, req: req.body , ...err }) - let errorMessage = {name: err.name, message: err.message}; + const errorMessage = {name: err.name, message: err.message}; ResponseHandler.errorResponse(errorMessage, req, res); }; -export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response) => { logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); diff --git a/api-service/src/v2/services/CipherService.ts b/api-service/src/v2/services/CipherService.ts index b0b86c25..5f85198d 100644 --- a/api-service/src/v2/services/CipherService.ts +++ b/api-service/src/v2/services/CipherService.ts @@ -1,5 +1,5 @@ -import crypto from 'crypto'; -import { config } from '../configs/Config'; +import crypto from "crypto"; +import { config } from "../configs/Config"; class CipherService { public encrypt(data: string) { @@ -8,10 +8,10 @@ class CipherService { config.encryption_config.encryption_key, "", ) - const toEncrypt = Buffer.from(data, 'utf8'); + const toEncrypt = Buffer.from(data, "utf8"); let encryptedString = cipher.update(toEncrypt); encryptedString = Buffer.concat([encryptedString, cipher.final()]) - return encryptedString.toString('base64'); + return encryptedString.toString("base64"); } public decrypt(data: string) { @@ -20,7 +20,7 @@ class CipherService { config.encryption_config.encryption_key, "", ) - const encryptedText = Buffer.from(data, 'base64'); + const encryptedText = Buffer.from(data, "base64"); let decryptedString = decipher.update(encryptedText); decryptedString = Buffer.concat([decryptedString, decipher.final()]) return decryptedString.toString(); diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 43bbcdf0..9de5bc6b 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -94,7 +94,7 @@ class DatasetService { migrateDraftDataset = async (datasetId: string, dataset: Model): Promise => { - let draftDataset : Record = { + const draftDataset : Record = { api_version: "v2" } const dataset_config:any = _.get(dataset, "dataset_config"); @@ -149,7 +149,7 @@ class DatasetService { createDraftDatasetFromLive = async (dataset: Model) => { - let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config:any = _.get(dataset, "dataset_config"); const api_version:any = _.get(dataset, "api_version"); if(api_version === "v1") { @@ -272,7 +272,7 @@ class DatasetService { const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -281,7 +281,7 @@ class DatasetService { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -292,7 +292,7 @@ class DatasetService { const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource.ingestion_spec, allFields, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -300,7 +300,7 @@ class DatasetService { const datasource = _.join([draftDataset.dataset_id,"events"], "_") return { - id: _.join([datasource,type], '_'), + id: _.join([datasource,type], "_"), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, diff --git a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts index 14a006d4..0eb13bbb 100644 --- a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts +++ b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts @@ -43,8 +43,8 @@ export class ConfigSuggestor { private processingConfig(conflicts: ConflictTypes[]): any { let dedupKeys = _.filter(conflicts, (o) => _.upperCase(o.formats.resolution["type"]) === "DEDUP").map(v => v.formats.property) let matchedDedupFields = [] - let dedupOrderProperty: string = "cardinality" - let dedupOrder: any = "desc" + const dedupOrderProperty: string = "cardinality" + const dedupOrder: any = "desc" if (!_.isUndefined(this.rollupInfo.summary)) { for (const key of Object.keys(this.rollupInfo.summary)) { if (!this.rollupInfo.summary[key].index) { diff --git a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts index 8a77344d..404ad24b 100644 --- a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts +++ b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts @@ -5,34 +5,34 @@ import moment from "moment"; import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException"; const DATE_FORMATS = [ - 'MM/DD/YYYY','DD/MM/YYYY', 'YYYY-MM-DD', 'YYYY-DD-MM', 'YYYY/MM/DD', - 'DD-MM-YYYY', 'MM-DD-YYYY', 'MM-DD-YYYY HH:mm:ss', 'YYYY/MM/DD HH:mm:ss', - 'YYYY-MM-DD HH:mm:ss', 'YYYY-DD-MM HH:mm:ss', 'DD/MM/YYYY HH:mm:ss', - 'DD-MM-YYYY HH:mm:ss', 'MM-DD-YYYY HH:mm:ss.SSS', 'YYYY-MM-DD HH:mm:ss.SSS', - 'YYYY-DD-MM HH:mm:ss.SSS', 'YYYY/MM/DD HH:mm:ss.SSS', 'DD/MM/YYYY HH:mm:ss.SSS', - 'DD-MM-YYYY HH:mm:ss.SSS', 'DD-MM-YYYYTHH:mm:ss.SSSZ', 'YYYY-MM-DDTHH:mm:ss.SSSZ', - 'YYYY-DD-MMTHH:mm:ss.SSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSZ', - 'YYYY-DD-MMTHH:mm:ss.SSS', 'YYYY/MM/DDTHH:mm:ss.SSS', 'DD/MM/YYYYTHH:mm:ss.SSS', - 'MM-DD-YYYYTHH:mm:ss.SSSZ', 'DD-MM-YYYYTHH:mm:ssZ', 'YYYY-MM-DDTHH:mm:ssZ', - 'YYYY-DD-MMTHH:mm:ssZ', 'YYYY/MM/DDTHH:mm:ssZ', 'DD/MM/YYYYTHH:mm:ssZ', 'MM-DD-YYYYTHH:mm:ssZ', - 'MM-DD-YYYYTHH:mm:ss', 'DD-MM-YYYYTHH:mm:ss', 'YYYY-MM-DDTHH:mm:ss', 'YYYY-DD-MMTHH:mm:ss', - 'YYYY/MM/DDTHH:mm:ss', 'DD/MM/YYYYTHH:mm:ss', 'DD-MM-YYYY HH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss.SSSZ', - 'YYYY-DD-MM HH:mm:ss.SSSZ', 'YYYY/MM/DD HH:mm:ss.SSSZ', 'DD/MM/YYYY HH:mm:ss.SSSZ', - 'MM-DD-YYYY HH:mm:ss.SSSZ', 'DD-MM-YYYY HH:mm:ssZ', 'YYYY-MM-DD HH:mm:ssZ', 'YYYY-DD-MM HH:mm:ssZ', - 'YYYY/MM/DD HH:mm:ssZ', 'DD/MM/YYYY HH:mm:ssZ', 'MM-DD-YYYY HH:mm:ssZ', 'DD-MM-YYYYTHH:mm:ss.SSSSSSZ', - 'YYYY-MM-DDTHH:mm:ss.SSSSSSZ', 'YYYY-DD-MMTHH:mm:ss.SSSSSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSSSSZ', - 'DD/MM/YYYYTHH:mm:ss.SSSSSSZ', 'MM-DD-YYYYTHH:mm:ss.SSSSSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSSSS', - 'YYYY-DD-MMTHH:mm:ss.SSSSSS', 'YYYY/MM/DDTHH:mm:ss.SSSSSS', 'YYYY-MM-DDTHH:mm:ss.SSSSSS', - 'MM-DD-YYYYTHH:mm:ss.SSSSSS', 'DD-MM-YYYYTHH:mm:ss.SSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSS', - 'YYYY-MM-DD HH:mm:ss.SSSSSS', 'YYYY-DD-MM HH:mm:ss.SSSSSS', 'YYYY/MM/DD HH:mm:ss.SSSSSS', - 'DD/MM/YYYY HH:mm:ss.SSSSSS', 'MM-DD-YYYY HH:mm:ss.SSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSZ', - 'YYYY-MM-DDTHH:mm:ss.SSSSSSSSSZ', 'YYYY-DD-MMTHH:mm:ss.SSSSSSSSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSSSSSSSZ', - 'DD/MM/YYYYTHH:mm:ss.SSSSSSSSSZ', 'MM-DD-YYYYTHH:mm:ss.SSSSSSSSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSSSSSSS', - 'YYYY-DD-MMTHH:mm:ss.SSSSSSSSS', 'YYYY/MM/DDTHH:mm:ss.SSSSSSSSS', 'YYYY-MM-DDTHH:mm:ss.SSSSSSSSS', - 'MM-DD-YYYYTHH:mm:ss.SSSSSSSSS', 'DD-MM-YYYYTHH:mm:ss.SSSSSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSSSS', - 'YYYY-MM-DD HH:mm:ss.SSSSSSSSS', 'YYYY-DD-MM HH:mm:ss.SSSSSSSSS', 'YYYY/MM/DD HH:mm:ss.SSSSSSSSS', - 'DD/MM/YYYY HH:mm:ss.SSSSSSSSS', 'MM-DD-YYYY HH:mm:ss.SSSSSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSSSSZ', - 'DD-MM-YYYYTHH:mm:ss.SSSSSSSSSZ', + "MM/DD/YYYY","DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", + "DD-MM-YYYY", "MM-DD-YYYY", "MM-DD-YYYY HH:mm:ss", "YYYY/MM/DD HH:mm:ss", + "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", + "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", + "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", + "YYYY-DD-MMTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ss.SSSZ", "DD/MM/YYYYTHH:mm:ss.SSSZ", + "YYYY-DD-MMTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss.SSS", "DD/MM/YYYYTHH:mm:ss.SSS", + "MM-DD-YYYYTHH:mm:ss.SSSZ", "DD-MM-YYYYTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ssZ", + "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", + "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", + "YYYY/MM/DDTHH:mm:ss", "DD/MM/YYYYTHH:mm:ss", "DD-MM-YYYY HH:mm:ss.SSSZ", "YYYY-MM-DD HH:mm:ss.SSSZ", + "YYYY-DD-MM HH:mm:ss.SSSZ", "YYYY/MM/DD HH:mm:ss.SSSZ", "DD/MM/YYYY HH:mm:ss.SSSZ", + "MM-DD-YYYY HH:mm:ss.SSSZ", "DD-MM-YYYY HH:mm:ssZ", "YYYY-MM-DD HH:mm:ssZ", "YYYY-DD-MM HH:mm:ssZ", + "YYYY/MM/DD HH:mm:ssZ", "DD/MM/YYYY HH:mm:ssZ", "MM-DD-YYYY HH:mm:ssZ", "DD-MM-YYYYTHH:mm:ss.SSSSSSZ", + "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", "YYYY-DD-MMTHH:mm:ss.SSSSSSZ", "YYYY/MM/DDTHH:mm:ss.SSSSSSZ", + "DD/MM/YYYYTHH:mm:ss.SSSSSSZ", "MM-DD-YYYYTHH:mm:ss.SSSSSSZ", "DD/MM/YYYYTHH:mm:ss.SSSSSS", + "YYYY-DD-MMTHH:mm:ss.SSSSSS", "YYYY/MM/DDTHH:mm:ss.SSSSSS", "YYYY-MM-DDTHH:mm:ss.SSSSSS", + "MM-DD-YYYYTHH:mm:ss.SSSSSS", "DD-MM-YYYYTHH:mm:ss.SSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSS", + "YYYY-MM-DD HH:mm:ss.SSSSSS", "YYYY-DD-MM HH:mm:ss.SSSSSS", "YYYY/MM/DD HH:mm:ss.SSSSSS", + "DD/MM/YYYY HH:mm:ss.SSSSSS", "MM-DD-YYYY HH:mm:ss.SSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSZ", + "YYYY-MM-DDTHH:mm:ss.SSSSSSSSSZ", "YYYY-DD-MMTHH:mm:ss.SSSSSSSSSZ", "YYYY/MM/DDTHH:mm:ss.SSSSSSSSSZ", + "DD/MM/YYYYTHH:mm:ss.SSSSSSSSSZ", "MM-DD-YYYYTHH:mm:ss.SSSSSSSSSZ", "DD/MM/YYYYTHH:mm:ss.SSSSSSSSS", + "YYYY-DD-MMTHH:mm:ss.SSSSSSSSS", "YYYY/MM/DDTHH:mm:ss.SSSSSSSSS", "YYYY-MM-DDTHH:mm:ss.SSSSSSSSS", + "MM-DD-YYYYTHH:mm:ss.SSSSSSSSS", "DD-MM-YYYYTHH:mm:ss.SSSSSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSSSS", + "YYYY-MM-DD HH:mm:ss.SSSSSSSSS", "YYYY-DD-MM HH:mm:ss.SSSSSSSSS", "YYYY/MM/DD HH:mm:ss.SSSSSSSSS", + "DD/MM/YYYY HH:mm:ss.SSSSSSSSS", "MM-DD-YYYY HH:mm:ss.SSSSSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSSSSZ", + "DD-MM-YYYYTHH:mm:ss.SSSSSSSSSZ", ]; export class SchemaInference { @@ -49,17 +49,17 @@ export class SchemaInference { if (extracted) { return this.inferSchema(extracted); } else { - throw new SchemaGenerationException('Unable to extract the batch data.', httpStatus.BAD_REQUEST); + throw new SchemaGenerationException("Unable to extract the batch data.", httpStatus.BAD_REQUEST); } } else { - throw new SchemaGenerationException('Extraction key not found.', httpStatus.BAD_REQUEST); + throw new SchemaGenerationException("Extraction key not found.", httpStatus.BAD_REQUEST); } }) } private validateEpoch(schema: any, sample: any, path: any) { Object.entries(sample).map(([key, value]) => { - if (value && typeof value == 'object') { + if (value && typeof value == "object") { this.validateEpoch(schema, value, `${path}.${key}.properties`) } const { isValidTimestamp, type } = this.isValidTimestamp(value); @@ -77,7 +77,7 @@ export class SchemaInference { isValidTimestamp(value: any) { const dataType = typeof value; switch (dataType) { - case 'string': + case "string": const epochRegex = /^\d+$/ig; if(epochRegex.test(value)){ const parsedValue = parseInt(value, 10); @@ -90,7 +90,7 @@ export class SchemaInference { isValidTimestamp: moment(value, DATE_FORMATS, true).isValid(), type: "date-time" } - case 'number': + case "number": // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds return { isValidTimestamp: value >= 946684800 && moment(value).isValid(), diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts index 14ee0e3d..4f0cb3ce 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts @@ -25,7 +25,7 @@ export class SchemaAnalyser { const result: FlattenSchema[] = _.flatten(this.schemas.map(element => { return this.flattenSchema(new Map(Object.entries(element))); })) - const conflicts = Object.entries(_.groupBy(result, 'path')).map(([key, value]) => { + const conflicts = Object.entries(_.groupBy(result, "path")).map(([key, value]) => { return this.getSchemaConflictTypes(this.getOccurance(value, key)) }) return _.filter(conflicts, obj => (!_.isEmpty(obj.schema) || !_.isEmpty(obj.required) || !_.isEmpty(obj.formats))) @@ -43,7 +43,7 @@ export class SchemaAnalyser { let schemaConflicts = this.findDataTypeConflicts(occuranceObj,) const requiredConflicts = (_.size(this.schemas) > this.minimumSchemas) ? this.findOptionalPropConflicts(occuranceObj) : {} const formatConflict = this.findFormatConflicts(occuranceObj) - if(_.size(_.keys(schemaConflicts)) > 0) { + if (_.size(_.keys(schemaConflicts)) > 0) { schemaConflicts = { ...schemaConflicts, path: updatedPath } } return { "schema": schemaConflicts, "required": requiredConflicts, "formats": formatConflict, "absolutePath": updatedPath } @@ -53,14 +53,14 @@ export class SchemaAnalyser { * Method to get the data type conflicts */ private findDataTypeConflicts(occurance: Occurance): Conflict { - if(_.includes(_.keys(occurance.dataType), "null") && _.size(occurance.dataType) === 1) { + if (_.includes(_.keys(occurance.dataType), "null") && _.size(occurance.dataType) === 1) { return { type: constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD, // Should be used only to return the name of field instead of path // property: Object.keys(occurance.property)[0], property: _.replace(Object.keys(occurance.path)[0], "$.", ""), conflicts: occurance.dataType, - resolution: { "value": occurance.dataType, "type": constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD }, + resolution: { "value": occurance.dataType, "type": constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD }, values: _.keys(occurance.dataType), severity: constants.SEVERITY["MUST-FIX"], path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), @@ -158,26 +158,26 @@ export class SchemaAnalyser { * Method to iterate over the schema object in a recursive and flatten the required properties */ public flattenSchema(sample: Map): FlattenSchema[] { - let array = new Array(); + const array: any[] = []; const recursive = (data: any, path: string, requiredProps: string[], schemaPath: string) => { _.map(data, (value, key) => { - let isMultipleTypes = ''; - if(_.has(value, 'anyOf')) isMultipleTypes = 'anyOf'; - if(_.has(value, 'oneOf')) isMultipleTypes = 'oneOf'; - if (_.isPlainObject(value) && (_.has(value, 'properties'))) { - array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) - recursive(value['properties'], `${path}.${key}`, value['required'], `${schemaPath}.properties.${key}`); - } else if(_.isPlainObject(value)) { - if (value.type === 'array') { - array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) - if (_.has(value, 'items') && _.has(value["items"], 'properties')) { - recursive(value["items"]['properties'], `${path}.${key}[*]`, value["items"]['required'], `${schemaPath}.properties.${key}.items`); + let isMultipleTypes = ""; + if (_.has(value, "anyOf")) isMultipleTypes = "anyOf"; + if (_.has(value, "oneOf")) isMultipleTypes = "oneOf"; + if (_.isPlainObject(value) && (_.has(value, "properties"))) { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + recursive(value["properties"], `${path}.${key}`, value["required"], `${schemaPath}.properties.${key}`); + } else if (_.isPlainObject(value)) { + if (value.type === "array") { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + if (_.has(value, "items") && _.has(value["items"], "properties")) { + recursive(value["items"]["properties"], `${path}.${key}[*]`, value["items"]["required"], `${schemaPath}.properties.${key}.items`); } - } else if(isMultipleTypes != '') { - array.push(this._flattenSchema(key, value[isMultipleTypes][0].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) - array.push(this._flattenSchema(key, value[isMultipleTypes][1].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + } else if (isMultipleTypes != "") { + array.push(this._flattenSchema(key, value[isMultipleTypes][0].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + array.push(this._flattenSchema(key, value[isMultipleTypes][1].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) } else { - array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) } } }) diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts index cc52309e..4bd19c91 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts @@ -4,7 +4,7 @@ export class SchemaArrayValidator { public validate(schemas: any) { _.map(schemas, (schema: any, index: number) => { Object.entries(schema).map(([schemaKey, schemaValue]) => { - if (typeof schemaValue === 'object') { + if (typeof schemaValue === "object") { this.handleNestedObject(index, `${schemaKey}`, schemaValue, schemas); } }); @@ -13,15 +13,15 @@ export class SchemaArrayValidator { } private checkForInvalidArray(value: any) { - if (_.has(value, 'items') && _.has(value, 'properties')) - _.unset(value, 'properties'); + if (_.has(value, "items") && _.has(value, "properties")) + _.unset(value, "properties"); } private handleNestedObject(index: any, path: string, value: any, schemas: any) { Object.entries(value).map(([nestedKey, nestedValue]: any) => { - if (typeof nestedValue === 'object') { + if (typeof nestedValue === "object") { this.handleNestedObject(index, `${path}.${nestedKey}`, nestedValue, schemas) - } else if (nestedValue.type === 'array' && (nestedValue.items != false)) { + } else if (nestedValue.type === "array" && (nestedValue.items != false)) { this.checkForInvalidArray(nestedValue); let isValidArray = true; if(_.isEqual(_.get(schemas[0], `${path}.${nestedKey}.type`), _.get(schemas[index], `${path}.${nestedKey}.type`))) { @@ -33,14 +33,14 @@ export class SchemaArrayValidator { if (!isValidArray) { this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) } - } else if (nestedValue.type === 'array' && (nestedValue.items == false)) { + } else if (nestedValue.type === "array" && (nestedValue.items == false)) { this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) } }) } private deleteItemsAndSetAdditionalProperties(schemas: any, path: string) { - _.map((schemas), (schema: any, index: number) => { + _.map((schemas), (schema: any) => { if (!isUndefined(_.get(schema, path))) { _.unset(schema, `${path}`) _.set(schema, `${path}.type`, "array"); diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts index 832ce7a5..99f15ea2 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts @@ -6,21 +6,21 @@ import { UniqueValues, FieldSchema, RollupSummary } from "../../types/SchemaMode export const generateRollupSummary = (uniqueValues: UniqueValues) => { const summary: RollupSummary = {}; Object.entries(uniqueValues).map(([field, value]) => { - let data: Record = {}; + const data: Record = {}; _.map(value, (item: string) => { if (!_.has(data, [field, item])) _.set(data, [field, item], 1); else data[field][item] += 1; }); const resultData: Record = {}; _.map(_.keys(data), (path: string) => { - Object.entries(data[path]).map(([key, value]: any) => { + Object.entries(data[path]).map(([, value]: any) => { const totalValue = _.sum(_.values(data[path])); const ratio = Math.round((value / totalValue) * 100); if (!_.has(resultData, path)) _.set(resultData, path, ratio); else if (ratio > _.get(resultData, path)) _.set(resultData, path, ratio); }); - let fieldName = parseSchemaPath(path); + const fieldName = parseSchemaPath(path); summary[fieldName] = { path: `$.${path}`, cardinality: 100 - _.get(resultData, path), diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts index ad3bb713..b71de416 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts @@ -13,7 +13,7 @@ export const dataMappingPaths = { "number": "number.store_format.number.jsonSchema", "object": "object.store_format.object.jsonSchema", "array": "array.store_format.array.jsonSchema", -} +} export class SchemaHandler { private typeToMethod = { @@ -52,23 +52,23 @@ export class SchemaHandler { private setNulltype(schema: any, conflict: ConflictTypes): any { const { absolutePath, schema: { resolution: { value } } } = conflict; const dataTypes: any = []; - _.forEach(DataMappings, (valueItem, keyItem) => { - _.forEach(_.get(valueItem, 'store_format'), (subValue, subKey) => { + _.forEach(DataMappings, (valueItem) => { + _.forEach(_.get(valueItem, "store_format"), (subValue) => { if (!_.find(dataTypes, ["type", subValue["jsonSchema"]])) dataTypes.push({ type: subValue["jsonSchema"] }) }) }); const arrivalDataTypes: any = _.keys(DataMappings).map((key: any) => ({ type: key })); - _.set(schema, `${absolutePath}.type`, 'null'); + _.set(schema, `${absolutePath}.type`, "null"); _.set(schema, `${absolutePath}.arrivalOneOf`, arrivalDataTypes); return _.set(schema, `${absolutePath}.oneof`, dataTypes); } private updateRequiredProp(schema: any, value: ConflictTypes): any { - const absolutePath = value.absolutePath.replace(value.required.property, value.required.property.replace('.', '$')) - const subStringArray: string[] = _.split(absolutePath, '.'); - const subString: string = _.join(_.slice(subStringArray, 0, subStringArray.length - 2), '.'); + const absolutePath = value.absolutePath.replace(value.required.property, value.required.property.replace(".", "$")) + const subStringArray: string[] = _.split(absolutePath, "."); + const subString: string = _.join(_.slice(subStringArray, 0, subStringArray.length - 2), "."); const path: string = _.isEmpty(subString) ? "required" : `${subString}.required` const requiredList: string[] = _.get(schema, path) const newProperty: string = value.required.property @@ -77,7 +77,7 @@ export class SchemaHandler { } private getArrivalSuggestions(schema: any, fieldData: any, property: any, type: string) { - let arrivalSuggestions: any = []; + const arrivalSuggestions: any = []; const types = _.get(fieldData, type); types && types.map((item: any) => { const storeFormat = _.get(dataMappingPaths, item.type); @@ -86,7 +86,7 @@ export class SchemaHandler { if (arrivalSuggestions.length > 0) _.set(schema, `${property}.arrivalOneOf`, arrivalSuggestions); return; - }; + } private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { const types = _.get(fieldData, type); @@ -105,11 +105,11 @@ export class SchemaHandler { const arrivalConflictExists = _.filter(suggestions, (suggestion) => _.has(suggestion, "arrivalConflict")); switch (true) { // Add arrival conflicts if there is arrival conflict in suggestions - case _.has(fieldData, 'oneof') && arrivalConflictExists.length > 0: - return this.getArrivalSuggestions(schema, fieldData, property, 'oneof') + case _.has(fieldData, "oneof") && arrivalConflictExists.length > 0: + return this.getArrivalSuggestions(schema, fieldData, property, "oneof") // Add arrival type if there are no arrival type conflicts case arrivalConflictExists.length === 0: - return this.getArrivalFormat(schema, fieldData, property, 'oneof') + return this.getArrivalFormat(schema, fieldData, property, "oneof") default: break; } @@ -126,22 +126,22 @@ export class SchemaHandler { } private checkForInvalidArray(value: any) { - if (_.has(value, 'items') && _.has(value, 'properties')) - _.unset(value, 'properties'); + if (_.has(value, "items") && _.has(value, "properties")) + _.unset(value, "properties"); } private updateMappings(schema: Map) { const recursive = (data: any) => { - _.map(data, (value, key) => { + _.map(data, (value) => { if (_.isPlainObject(value)) { - if ((_.has(value, 'properties'))) { - recursive(value['properties']); + if ((_.has(value, "properties"))) { + recursive(value["properties"]); } - if (value.type === 'array') { - if (_.has(value, 'items') && _.has(value["items"], 'properties')) { - recursive(value["items"]['properties']); + if (value.type === "array") { + if (_.has(value, "items") && _.has(value["items"], "properties")) { + recursive(value["items"]["properties"]); } - if (_.has(value, 'items') && _.has(value, 'properties')) + if (_.has(value, "items") && _.has(value, "properties")) this.checkForInvalidArray(value); this.updateStoreType(value, _.get(value, "type")); } else { diff --git a/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts index bb87fe32..fcaba4a5 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts @@ -6,7 +6,7 @@ import { SchemaSuggestionTemplate } from "./Template" export class SuggestionTemplate { public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { - return _.map(sample, (value, key) => { + return _.map(sample, (value) => { const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) const requiredSuggestions = this.getRequiredMessageTemplate(value.required) const formatSuggestions = this.getPropertyFormatTemplate(value.formats) @@ -24,7 +24,7 @@ export class SuggestionTemplate { message = SchemaSuggestionTemplate.getSchemaNullTypeMessage(object.conflicts, object.property); advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.ADVICE; } else { - let { conflictMessage, arrivalFormatMessage } = SchemaSuggestionTemplate.getSchemaDataTypeMessage(object.conflicts, object.property); + const { conflictMessage, arrivalFormatMessage } = SchemaSuggestionTemplate.getSchemaDataTypeMessage(object.conflicts, object.property); message = conflictMessage; arrival_format_message = arrivalFormatMessage; advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.ADVICE; diff --git a/api-service/src/v2/services/SchemaGenerateService/Template.ts b/api-service/src/v2/services/SchemaGenerateService/Template.ts index 49c86228..9beb3a46 100644 --- a/api-service/src/v2/services/SchemaGenerateService/Template.ts +++ b/api-service/src/v2/services/SchemaGenerateService/Template.ts @@ -71,7 +71,7 @@ export const SchemaSuggestionTemplate = { updatedConflicts[types[0]] = value; } }); - let response: Record = { + const response: Record = { conflictMessage: _.template( `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.MESSAGE} at property: '${property}'. The property type <% _.map(conflicts, (value, key, list) => { %><%= key %>: <%= value %> time(s)<%= _.last(list) === value ? '' : ', ' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts }), arrivalFormatMessage: null, diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index 838538b7..dc4afbfc 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -12,31 +12,31 @@ class BaseTableGenerator { */ flattenSchema = (dataSchema: Record, type: string) : Record[] => { - let properties: Record[] = [] + const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { _.mapKeys(schema, function(value, parentKey) { - const newKey = (prev) ? _.join([prev, parentKey], '.') : parentKey; - const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], '') : _.join(["$.['", parentKey, "']"], ''); - switch(value['type']) { - case 'object': - flatten(_.get(value, 'properties'), newKey, newExpr); + const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; + const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); + switch(value["type"]) { + case "object": + flatten(_.get(value, "properties"), newKey, newExpr); break; - case 'array': - if(type === "druid" && _.get(value, 'items.type') == 'object' && _.get(value, 'items.properties')) { - _.mapKeys(_.get(value, 'items.properties'), function(value, childKey) { - const objChildKey = _.join([newKey, childKey], '.') - properties.push(_.merge(_.pick(value, ['type', 'arrival_format', 'is_deleted']), {expr: _.join([newExpr,"[*].['",childKey,"']"], ''), name: objChildKey, data_type: 'array'})) + case "array": + if(type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { + _.mapKeys(_.get(value, "items.properties"), function(value, childKey) { + const objChildKey = _.join([newKey, childKey], ".") + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), {expr: _.join([newExpr,"[*].['",childKey,"']"], ""), name: objChildKey, data_type: "array"})) }) } else { - properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr+'[*]', name: newKey, type: _.get(value, 'items.type')})) + properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), {expr: newExpr+"[*]", name: newKey, type: _.get(value, "items.type")})) } break; default: - properties.push(_.merge(_.pick(value, ['type', 'arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), {expr: newExpr, name: newKey})) } }); } - flatten(_.get(dataSchema, 'properties'), undefined, undefined) + flatten(_.get(dataSchema, "properties"), undefined, undefined) return properties } @@ -58,8 +58,8 @@ class BaseTableGenerator { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); const properties = instance.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { - _.set(prop, 'name', _.join([denormField.denorm_out_field, prop.name], '.')); - _.set(prop, 'expr', _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); return prop; }); dataFields.push(...transformProps); @@ -182,12 +182,12 @@ class TableGenerator extends BaseTableGenerator { getHudiIngestionSpecForUpdate = (dataset: Record, existingHudiSpec: Record, allFields: Record[], datasourceRef: string) => { - let newHudiSpec = this.getHudiIngestionSpecForCreate(dataset, allFields, datasourceRef) + const newHudiSpec = this.getHudiIngestionSpecForCreate(dataset, allFields, datasourceRef) const newColumnSpec = newHudiSpec.schema.columnSpec; - let oldColumnSpec = existingHudiSpec.schema.columnSpec; - let currIndex = _.get(_.maxBy(oldColumnSpec, 'index'), 'index') as unknown as number - const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, 'name'); + const oldColumnSpec = existingHudiSpec.schema.columnSpec; + let currIndex = _.get(_.maxBy(oldColumnSpec, "index"), "index") as unknown as number + const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, "name"); if(_.size(newColumns) > 0) { _.each(newColumns, (col) => { oldColumnSpec.push({ @@ -197,7 +197,7 @@ class TableGenerator extends BaseTableGenerator { }) }) } - _.set(newHudiSpec, 'schema.columnSpec', oldColumnSpec) + _.set(newHudiSpec, "schema.columnSpec", oldColumnSpec) return newHudiSpec; } @@ -227,10 +227,10 @@ class TableGenerator extends BaseTableGenerator { } private getHudiColumnType = (field: Record) : string => { - if(field.data_type === 'array' && field.arrival_format !== 'array') { + if(field.data_type === "array" && field.arrival_format !== "array") { return "array"; } - if(field.data_type === 'array' && field.arrival_format === 'array') { + if(field.data_type === "array" && field.arrival_format === "array") { switch(field.type) { case "string": return "array" diff --git a/api-service/src/v2/types/ConfigModels.ts b/api-service/src/v2/types/ConfigModels.ts index 320cbb32..a03ec5f4 100644 --- a/api-service/src/v2/types/ConfigModels.ts +++ b/api-service/src/v2/types/ConfigModels.ts @@ -1,5 +1,5 @@ import { IngestionConfig } from "./IngestionModels"; -import { IDataSourceRules, IRules } from "./QueryModels"; +import { IDataSourceRules } from "./QueryModels"; export interface ExtractionConfig { is_batch_event: boolean; From 0e1b392f8e2924fbaa9017bedfcdf6230bf1f267 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 21 Jul 2024 22:14:21 +0530 Subject: [PATCH 044/311] #OBS-I116: lint fixes --- .../GenerateDataSchema/GenerateDataSchema.ts | 4 +- api-service/src/v2/middlewares/errors.ts | 2 +- .../SchemaGenerateService/ConfigSuggester.ts | 4 +- .../DataSchemaService.ts | 2 +- .../SchemaGenerateService/SchemaAnalyser.ts | 12 +-- .../SchemaGenerateService/SchemaHandler.ts | 6 +- api-service/src/v2/services/TableGenerator.ts | 81 +++++++++---------- 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts index 3d16f5b2..a37e5d13 100644 --- a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -80,7 +80,7 @@ const checkJsonSchema = (sample: Map): boolean => { const removeNonIndexColumns = (schema: any) => { if (schema.properties) { - Object.entries(schema.properties).map(([key, property]: any) => { + Object.entries(schema.properties).map(([, property]: any) => { _.unset(schema, "required"); removeNonIndexColumns(property) }); @@ -95,7 +95,7 @@ const removeNonIndexColumns = (schema: any) => { const removeFormats = (schema: any) => { if (schema.properties) { - Object.entries(schema.properties).map(([key, property]: any) => { + Object.entries(schema.properties).map(([, property]: any) => { // Removing format to avoid schema validation issues const isDateTypeField = ["date-time", "date", "epoch"].includes((property as any).format); if (isDateTypeField && _.get(property, "data_type") === "string") { diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index 2ca96caf..b15d561a 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -1,4 +1,4 @@ -import { NextFunction, Request, Response } from "express"; +import { Request, Response } from "express"; import logger from "../logger"; import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; diff --git a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts index 0eb13bbb..8834129d 100644 --- a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts +++ b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts @@ -31,12 +31,12 @@ export class ConfigSuggestor { private analyzeConflicts(conflicts: ConflictTypes[]): DataSetConfig { const typeFormatsConflict: ConflictTypes[] = _.filter(conflicts, (o) => !_.isEmpty(o.formats)); - const ingestionConfig: IngestionConfig = this.ingestionConfig(typeFormatsConflict) + const ingestionConfig: IngestionConfig = this.ingestionConfig() const processingConfig: DatasetProcessing = this.processingConfig(typeFormatsConflict) return { "indexConfiguration": ingestionConfig, "processing": processingConfig } } - private ingestionConfig(conflicts: ConflictTypes[]): any { + private ingestionConfig(): any { return { "index": Object.assign(ingestionConfig.indexCol), "rollupSuggestions": this.rollupInfo }; } diff --git a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts index 404ad24b..67bde9e7 100644 --- a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts +++ b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts @@ -76,9 +76,9 @@ export class SchemaInference { isValidTimestamp(value: any) { const dataType = typeof value; + const epochRegex = /^\d+$/ig; switch (dataType) { case "string": - const epochRegex = /^\d+$/ig; if(epochRegex.test(value)){ const parsedValue = parseInt(value, 10); // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts index 4f0cb3ce..a3a24b1d 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts @@ -25,8 +25,8 @@ export class SchemaAnalyser { const result: FlattenSchema[] = _.flatten(this.schemas.map(element => { return this.flattenSchema(new Map(Object.entries(element))); })) - const conflicts = Object.entries(_.groupBy(result, "path")).map(([key, value]) => { - return this.getSchemaConflictTypes(this.getOccurance(value, key)) + const conflicts = Object.entries(_.groupBy(result, "path")).map(([, value]) => { + return this.getSchemaConflictTypes(this.getOccurance(value)) }) return _.filter(conflicts, obj => (!_.isEmpty(obj.schema) || !_.isEmpty(obj.required) || !_.isEmpty(obj.formats))) } @@ -123,7 +123,7 @@ export class SchemaAnalyser { */ private findOptionalPropConflicts(occurance: Occurance): Conflict { const maxOccurance: number = 1 - const requiredCount = _.map(occurance.property, (value, key) => { + const requiredCount = _.map(occurance.property, (value) => { return value })[0] @@ -148,9 +148,9 @@ export class SchemaAnalyser { * * Method to get the occurance of the given key from the given object */ - private getOccurance(arrayOfObjects: object[], key: string): Occurance { - const result = _(arrayOfObjects).flatMap(obj => _.toPairs(obj)).groupBy(([key, value]) => key) - .mapValues(group => _.countBy(group, ([key, value]) => value)).value(); + private getOccurance(arrayOfObjects: object[]): Occurance { + const result = _(arrayOfObjects).flatMap(obj => _.toPairs(obj)).groupBy(([key]) => key) + .mapValues(group => _.countBy(group, ([, value]) => value)).value(); return { property: result.property, dataType: result.dataType, isRequired: result.isRequired, path: result.path, absolutePath: result.absolutePath, format: result.formate }; } diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts index b71de416..45d76c9c 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts @@ -39,18 +39,18 @@ export class SchemaHandler { } private updateDataTypes(schema: any, conflict: ConflictTypes): any { - const { absolutePath, schema: { resolution: { value } } } = conflict; + const { absolutePath, schema: { resolution } } = conflict; return _.set(schema, `${absolutePath}`, { ...schema[absolutePath], ...{ - type: conflict.schema.resolution["value"], + type: resolution.value, oneof: conflict.schema.values.map(key => ({ type: key })), } }); } private setNulltype(schema: any, conflict: ConflictTypes): any { - const { absolutePath, schema: { resolution: { value } } } = conflict; + const { absolutePath } = conflict; const dataTypes: any = []; _.forEach(DataMappings, (valueItem) => { _.forEach(_.get(valueItem, "store_format"), (subValue) => { diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index dc4afbfc..eb3065ae 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -10,29 +10,29 @@ class BaseTableGenerator { * @param dataSchema * @returns properties Record[] */ - flattenSchema = (dataSchema: Record, type: string) : Record[] => { + flattenSchema = (dataSchema: Record, type: string): Record[] => { const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { - _.mapKeys(schema, function(value, parentKey) { + _.mapKeys(schema, function (value, parentKey) { const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); - switch(value["type"]) { - case "object": + switch (value["type"]) { + case "object": flatten(_.get(value, "properties"), newKey, newExpr); break; case "array": - if(type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { - _.mapKeys(_.get(value, "items.properties"), function(value, childKey) { + if (type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { + _.mapKeys(_.get(value, "items.properties"), function (value, childKey) { const objChildKey = _.join([newKey, childKey], ".") - properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), {expr: _.join([newExpr,"[*].['",childKey,"']"], ""), name: objChildKey, data_type: "array"})) + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), { expr: _.join([newExpr, "[*].['", childKey, "']"], ""), name: objChildKey, data_type: "array" })) }) } else { - properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), {expr: newExpr+"[*]", name: newKey, type: _.get(value, "items.type")})) + properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), { expr: newExpr + "[*]", name: newKey, type: _.get(value, "items.type") })) } break; default: - properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), {expr: newExpr, name: newKey})) + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), { expr: newExpr, name: newKey })) } }); } @@ -51,12 +51,11 @@ class BaseTableGenerator { getAllFields = async (dataset: Record, type: string): Promise[]> => { const { data_schema, denorm_config, transformations_config } = dataset - const instance = this; - let dataFields = instance.flattenSchema(data_schema, type); + let dataFields = this.flattenSchema(data_schema, type); if (!_.isEmpty(denorm_config.denorm_fields)) { for (const denormField of denorm_config.denorm_fields) { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); - const properties = instance.flattenSchema(denormDataset.data_schema, type); + const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); @@ -85,7 +84,7 @@ class BaseTableGenerator { class TableGenerator extends BaseTableGenerator { getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { - + const { dataset_config, router_config } = dataset return { "type": "kafka", @@ -109,22 +108,21 @@ class TableGenerator extends BaseTableGenerator { } } } - + private getDruidDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { const dataFields = _.cloneDeep(allFields); - if(partitionKey) { // Move the partition column to the top of the dimensions - const partitionCol = _.remove(dataFields, {name: partitionKey}) - if(partitionCol && _.size(partitionCol) > 0) { + if (partitionKey) { // Move the partition column to the top of the dimensions + const partitionCol = _.remove(dataFields, { name: partitionKey }) + if (partitionCol && _.size(partitionCol) > 0) { dataFields.unshift(partitionCol[0]) } } - _.remove(dataFields, {name: timestampKey}) - const instance = this; + _.remove(dataFields, { name: timestampKey }) return _.union( _.map(dataFields, (field) => { return { - "type": instance.getDruidDimensionType(field.data_type), + "type": this.getDruidDimensionType(field.data_type), "name": field.name } }), @@ -132,7 +130,7 @@ class TableGenerator extends BaseTableGenerator { ) } - private getDruidDimensionType = (data_type: string):string => { + private getDruidDimensionType = (data_type: string): string => { switch (data_type) { case "number": return "double"; case "integer": return "long"; @@ -188,7 +186,7 @@ class TableGenerator extends BaseTableGenerator { const oldColumnSpec = existingHudiSpec.schema.columnSpec; let currIndex = _.get(_.maxBy(oldColumnSpec, "index"), "index") as unknown as number const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, "name"); - if(_.size(newColumns) > 0) { + if (_.size(newColumns) > 0) { _.each(newColumns, (col) => { oldColumnSpec.push({ "type": col.type, @@ -201,17 +199,16 @@ class TableGenerator extends BaseTableGenerator { return newHudiSpec; } - private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string) : Record[] => { + private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { - const instance = this; const dataFields = _.cloneDeep(allFields); - _.remove(dataFields, {name: primaryKey}) - _.remove(dataFields, {name: partitionKey}) - _.remove(dataFields, {name: timestampKey}) + _.remove(dataFields, { name: primaryKey }) + _.remove(dataFields, { name: partitionKey }) + _.remove(dataFields, { name: timestampKey }) let index = 1; - const transformFields = _.map(dataFields, (field) => { + const transformFields = _.map(dataFields, (field) => { return { - "type": instance.getHudiColumnType(field), + "type": this.getHudiColumnType(field), "name": field.name, "index": index++ } @@ -226,12 +223,12 @@ class TableGenerator extends BaseTableGenerator { return transformFields; } - private getHudiColumnType = (field: Record) : string => { - if(field.data_type === "array" && field.arrival_format !== "array") { + private getHudiColumnType = (field: Record): string => { + if (field.data_type === "array" && field.arrival_format !== "array") { return "array"; } - if(field.data_type === "array" && field.arrival_format === "array") { - switch(field.type) { + if (field.data_type === "array" && field.arrival_format === "array") { + switch (field.type) { case "string": return "array" case "number": @@ -244,11 +241,11 @@ class TableGenerator extends BaseTableGenerator { return "array" } } - switch(field.arrival_format) { + switch (field.arrival_format) { case "text": return "string" case "number": - switch(field.data_type) { + switch (field.data_type) { case "integer": return "int" case "epoch": @@ -260,7 +257,7 @@ class TableGenerator extends BaseTableGenerator { case "long": return "long" default: - return "double" + return "double" } case "integer": return "int" @@ -271,13 +268,15 @@ class TableGenerator extends BaseTableGenerator { } } - private getHudiFields = (allFields: Record[]) : Record[] => { + private getHudiFields = (allFields: Record[]): Record[] => { + const regexString = "[\\[\\]'\\*]"; + const regex = new RegExp(regexString, "g"); return _.union( _.map(allFields, (field) => { return { type: "path", - expr: _.replace(field.expr, /[\[\]'\*]/g, ""), + expr: _.replace(field.expr, regex, ""), name: field.name } }), @@ -285,15 +284,15 @@ class TableGenerator extends BaseTableGenerator { ) } - private getPrimaryKey = (dataset: Record) : string => { + private getPrimaryKey = (dataset: Record): string => { return dataset.dataset_config.keys_config.data_key; } - private getHudiPartitionKey = (dataset: Record) : string => { + private getHudiPartitionKey = (dataset: Record): string => { return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; } - private getTimestampKey = (dataset: Record) : string => { + private getTimestampKey = (dataset: Record): string => { return dataset.dataset_config.keys_config.timestamp_key; } } From d206d87fa07c07f3abc919e2e83e8d9a676e7c38 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 21 Jul 2024 22:43:55 +0530 Subject: [PATCH 045/311] #OBS-I116: Dataset status transition test cases --- api-service/src/v2/middlewares/errors.ts | 6 +++--- .../DatasetStatusTransition/DatasetDelete.spec.ts | 9 ++++++++- .../DatasetStatusTransition/DatasetLive.spec.ts | 2 +- .../DatasetReadyToPublish.spec.ts | 2 +- .../DatasetStatusTransition/DatasetRetire.spec.ts | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index b15d561a..96c8069e 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -1,10 +1,10 @@ -import { Request, Response } from "express"; +import { NextFunction, Request, Response } from "express"; import logger from "../logger"; import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; -export const errorHandler = (err: Error, req: Request, res: Response) => { +export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { logger.error({ path: req.url, req: req.body , ...err }) const errorMessage = {name: err.name, message: err.message}; @@ -12,7 +12,7 @@ export const errorHandler = (err: Error, req: Request, res: Response) => { }; -export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response) => { +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts index 29a8584d..f3de6b4f 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts @@ -10,6 +10,7 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); @@ -40,6 +41,12 @@ describe("DATASET STATUS TRANSITION DELETE", () => { chai.spy.on(DatasetDraft, "destroy", () => { return Promise.resolve({}) }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) chai .request(app) .post("/v2/datasets/status-transition") @@ -71,7 +78,7 @@ describe("DATASET STATUS TRANSITION DELETE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to delete") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 4623a0e3..d7dbfb36 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -59,7 +59,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to perform status transition to live") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 83173c68..b8f83879 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -59,7 +59,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to perform status transition to ready to publish") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 21d721d3..021b7305 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -166,7 +166,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to retire") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }) From 0c3edb7a4f1657973dfa8025e9444eb81d6b46d0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 5 Aug 2024 11:25:39 +0530 Subject: [PATCH 046/311] #OBS-I116: feat: Test cases and linting fixes --- .github/workflows/pull_request.yaml | 1 - api-service/.eslintignore | 2 +- api-service/package.json | 5 +-- api-service/src/configs/Config.ts | 2 +- .../src/connections/grafanaConnection.ts | 2 +- api-service/src/controllers/Alerts/Alerts.ts | 16 +++---- api-service/src/controllers/Alerts/Metric.ts | 20 ++++----- api-service/src/controllers/Alerts/Silence.ts | 18 ++++---- .../controllers/DatasetCopy/DatasetCopy.ts | 6 +-- .../DatasetCopy/DatasetCopyHelper.ts | 6 +-- .../DatasetImport/DatasetImport.ts | 6 +-- .../DatasetImport/DatasetImportHelper.ts | 4 +- .../controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 12 +++--- .../NotificationChannel/Notification.ts | 20 ++++----- api-service/src/helpers/ResponseHandler.ts | 2 +- api-service/src/models/Alert.ts | 2 +- api-service/src/models/Metric.ts | 2 +- api-service/src/models/Notification.ts | 2 +- api-service/src/models/Silence.ts | 2 +- api-service/src/routes/AlertsRouter.ts | 2 +- .../src/services/DatasetHealthService.ts | 6 +-- api-service/src/services/DatasetService.ts | 6 +-- api-service/src/services/HealthService.ts | 4 +- api-service/src/services/WrapperService.ts | 1 - api-service/src/services/fs.ts | 6 +-- .../managers/grafana/alert/helpers/index.ts | 18 ++++---- .../services/managers/grafana/alert/index.ts | 26 ++++++------ .../src/services/managers/grafana/index.ts | 6 +-- .../grafana/notification/channels/email.ts | 8 ++-- .../grafana/notification/channels/index.ts | 8 ++-- .../grafana/notification/channels/slack.ts | 7 ++-- .../grafana/notification/channels/teams.ts | 7 ++-- .../grafana/notification/helpers/index.ts | 20 ++++----- .../managers/grafana/notification/index.ts | 3 +- .../grafana/silences/helpers/index.ts | 2 +- api-service/src/services/managers/index.ts | 14 +++---- .../managers/prometheus/alert/index.ts | 12 +++--- .../src/services/managers/prometheus/index.ts | 6 +-- .../managers/prometheus/notification/index.ts | 8 ++-- .../managers/prometheus/silences/index.ts | 10 ++--- .../DataIngestTest/DataIngestionTest.spec.ts | 19 ++++----- .../DataOutTest/DataQueryTest.spec.ts | 8 ++-- .../DatasetManagement/DataOutTest/Fixtures.ts | 22 +++++----- .../DatasetCreate/DatasetCreate.spec.ts | 3 +- .../DatasetCreate/Fixtures.ts | 1 - .../DatasetList/DatasetList.spec.ts | 5 +-- .../DatasetRead/DatasetRead.spec.ts | 20 ++++----- .../DatasetDelete.spec.ts | 4 +- .../DatasetLive.spec.ts | 4 +- .../DatasetReadyToPublish.spec.ts | 4 +- .../DatasetRetire.spec.ts | 4 +- .../DatasetStatusTransition.spec.ts | 4 +- .../DatasetUpdate/DatasetDedup.spec.ts | 4 +- .../DatasetUpdate/DatasetDenorm.spec.ts | 2 +- .../DatasetUpdate/DatasetExtraction.spec.ts | 4 +- .../DatasetUpdate/DatasetTags.spec.ts | 2 +- .../DatasetTransformation.spec.ts | 2 +- .../DatasetUpdate/DatasetUpdate.spec.ts | 2 +- .../DatasetUpdate/DatasetValidation.spec.ts | 4 +- .../GenerateSignedURL/Fixtures.ts | 16 +++---- .../GenerateSignedURL.spec.ts | 4 +- .../CreateTemplate/CreateTemplate.spec.ts | 6 +-- .../DeleteTemplate/DeleteTemplate.spec.ts | 8 ++-- .../ListTemplates/ListTemplates.spec.ts | 6 +-- .../ReadTemplate/ReadTemplate.spec.ts | 18 ++++---- .../TemplateQuerying/TemplateQuerying.spec.ts | 42 +++++++++---------- .../UpdateTemplate/UpdateTemplate.spec.ts | 6 +-- .../SqlWrapper/SqlWrapper.spec.ts | 4 +- 69 files changed, 265 insertions(+), 275 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 12e967e2..8b4fc54d 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -24,5 +24,4 @@ jobs: cd api-service npm install npm run actions:test - npm run actions:test:v2 npm run lint \ No newline at end of file diff --git a/api-service/.eslintignore b/api-service/.eslintignore index f56aadad..e528fbc0 100644 --- a/api-service/.eslintignore +++ b/api-service/.eslintignore @@ -4,6 +4,6 @@ docs coverage @types .nyc_output -src/v2/tests +src/tests src/v1 dist \ No newline at end of file diff --git a/api-service/package.json b/api-service/package.json index f760df95..1360ecf6 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -5,9 +5,8 @@ "main": "dist/app.js", "scripts": { "start": "ts-node ./src/app.ts", - "test": "source .env.test && nyc mocha ./src/v1/test/*.spec.ts --exit && nyc mocha ./src/v2/tests/**/*.spec.ts --exit", - "actions:test": "nyc mocha ./src/v1/test/*.spec.ts --exit", - "actions:test:v2": "nyc mocha ./src/v2/tests/**/*.spec.ts --exit", + "test": "source .env.test && nyc mocha ./src/tests/**/*.spec.ts --exit", + "actions:test": "nyc mocha ./src/test/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", "package": "npm run build && cd dist && npm pack . && cd ..", "lint": "eslint . --ext .ts", diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 28f83b1e..38fe624c 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -109,7 +109,7 @@ export const config = { "encryption_algorithm": process.env.encryption_algorithm || "aes-256-ecb", }, "grafana_config": { - "dialect": process.env.dialet || 'postgres', + "dialect": process.env.dialet || "postgres", "url": process.env.grafana_url || "http://localhost:8000", "access_token": process.env.grafana_token || "" } diff --git a/api-service/src/connections/grafanaConnection.ts b/api-service/src/connections/grafanaConnection.ts index 76df0cea..b70e6009 100644 --- a/api-service/src/connections/grafanaConnection.ts +++ b/api-service/src/connections/grafanaConnection.ts @@ -5,6 +5,6 @@ const grafanaHttpClient = axios.create({ baseURL: config.grafana_config.url }); -grafanaHttpClient.defaults.headers.common['Authorization'] = config.grafana_config.access_token; +grafanaHttpClient.defaults.headers.common["Authorization"] = config.grafana_config.access_token; export { grafanaHttpClient }; \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 056a2475..278ac586 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -17,9 +17,9 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); } catch (error: any) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } @@ -44,7 +44,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const transformAlerts = async (alertModel: any) => { const alert = alertModel?.toJSON(); - const status = _.get(alert, 'status'); + const status = _.get(alert, "status"); if (status !== "live") return alert; return getAlertsMetadata(alert); } @@ -108,13 +108,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: 'draft' }, { where: { id: alertId } }); + await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } diff --git a/api-service/src/controllers/Alerts/Metric.ts b/api-service/src/controllers/Alerts/Metric.ts index 212d5b13..6f80f16f 100644 --- a/api-service/src/controllers/Alerts/Metric.ts +++ b/api-service/src/controllers/Alerts/Metric.ts @@ -10,15 +10,15 @@ const telemetryObject = { type: "metric", ver: "1.0.0" }; const createMetricHandler = async (req: Request, res: Response, next: NextFunction) => { try { - const { component } = req?.body; + const { component } = req.body; const transformComponent = _.toLower(component); const metricsBody = await Metrics.create({ ...(req.body), component: transformComponent }); updateTelemetryAuditEvent({ request: req, object: { id: metricsBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: metricsBody.dataValues.id } }); } catch (error: any) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } @@ -30,7 +30,7 @@ const listMetricsHandler = async (req: Request, res: Response, next: NextFunctio const metricsPayload = await Metrics.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { metrics: metricsPayload, count: metricsPayload.length } }); } catch (error) { - const errorMessage = _.get(error, 'message') + const errorMessage = _.get(error, "message") next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } } @@ -50,9 +50,9 @@ const updateMetricHandler = async (req: Request, res: Response, next: NextFuncti }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); } catch (error) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } @@ -66,7 +66,7 @@ const deleteMetricHandler = async (req: Request, res: Response, next: NextFuncti await record.destroy(); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); } catch (error) { - const errorMessage = _.get(error, 'message') + const errorMessage = _.get(error, "message") next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } } @@ -78,7 +78,7 @@ const deleteMultipleMetricHandler = async (req: Request, res: Response, next: Ne await Metrics.destroy({ where: filters }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} }); } catch (error) { - const errorMessage = _.get(error, 'message') + const errorMessage = _.get(error, "message") next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } } diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index c578dcbc..32e2c531 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -18,8 +18,8 @@ const createHandler = async (request: Request, response: Response, next: NextFun const grafanaResponse = await createSilence(payload); if (!grafanaResponse) return next({ message: httpStatus[httpStatus.INTERNAL_SERVER_ERROR], statusCode: httpStatus.INTERNAL_SERVER_ERROR }) - let start_date = new Date(startDate); - let end_date = new Date(endDate); + const start_date = new Date(startDate); + const end_date = new Date(endDate); const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, @@ -31,7 +31,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: sileneResponse.dataValues.id } }) } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -45,11 +45,11 @@ const transformSilences = async (silenceModel: any) => { const listHandler = async (request: Request, response: Response, next: NextFunction) => { try { const silences = await Silence.findAll(); - const count = _.get(silences, 'length'); + const count = _.get(silences, "length"); const transformedSilences = await Promise.all(silences.map(transformSilences)); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { transformedSilences, ...(count && { count }) } }); } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -62,7 +62,7 @@ const fetchHandler = async (request: Request, response: Response, next: NextFunc if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: transformedSilence }); } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -86,7 +86,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -97,13 +97,13 @@ const deleteHandler = async (request: Request, response: Response, next: NextFun const silenceModel = await Silence.findOne({ where: { id } }); if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); const silenceObject = silenceModel?.toJSON(); - if (silenceObject?.status === 'expired') return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST }); + if (silenceObject?.status === "expired") return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST }); await deleteSilence(silenceObject); await silenceModel.destroy(); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }) } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index b33b614e..9308dd71 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -18,7 +18,7 @@ const validateRequest = (req: Request) => { } } -const fetchDataset = async (req: Request, newDatasetId: string) => { +const fetchDataset = async (req: Request) => { const datasetId = _.get(req, "body.request.source.datasetId"); const isLive = _.get(req, "body.request.source.isLive"); @@ -39,10 +39,10 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); - const dataset = await fetchDataset(req, newDatasetId); + const dataset = await fetchDataset(req); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { - if (err?.name === 'SequelizeUniqueConstraintError') { + if (err?.name === "SequelizeUniqueConstraintError") { throw obsrvError(newDatasetId, "DATASET_ALREADY_EXISTS", `Dataset with id ${newDatasetId} already exists`, "BAD_REQUEST", 400); } throw obsrvError(newDatasetId, "DATASET_COPY_FAILURE", `Failed to clone dataset`, "INTERNAL_SERVER_ERROR", 500); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index de4c8087..7b50626c 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -6,13 +6,13 @@ const version = defaultDatasetConfig.version; export const updateRecords = (datasetRecord: Record, newDatasetId: string): void => { const dataset_id = newDatasetId; - _.set(datasetRecord, 'api_version', "v2") - _.set(datasetRecord, 'status', DatasetStatus.Draft) + _.set(datasetRecord, "api_version", "v2") + _.set(datasetRecord, "status", DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) - _.set(datasetRecord, 'version', version); + _.set(datasetRecord, "version", version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) _.set(datasetRecord, "router_config", { topic: newDatasetId }) } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 3f5ef29f..e8a0b033 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -25,9 +25,9 @@ const datasetImport = async (req: Request, res: Response) => { const importDataset = async (dataset: Record, overwrite: string | any) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) - if (response?.name === 'SequelizeUniqueConstraintError') { + if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { - const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(err=>{ + const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) return _.omit(overwriteRes, ["message"]) @@ -42,7 +42,7 @@ const importDataset = async (dataset: Record, overwrite: string | a const getResponseData = (ignoredConfigs: Record) => { const { ignoredConnectors, ignoredTransformations, ignoredDenorms } = ignoredConfigs; let successMsg = "Dataset is imported successfully"; - let partialIgnored: Record = {}; + const partialIgnored: Record = {}; if (ignoredConnectors.length || ignoredTransformations.length || ignoredDenorms.length) { successMsg = "Dataset is partially imported"; diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 9388ab21..b3ccb9bc 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -34,7 +34,7 @@ export const datasetImportValidation = async (payload: Record): Pro throw obsrvError("", "DATASET_IMPORT_INVALID_CONFIGS", isRequestValid.message, "BAD_REQUEST", 400) } - let datasetConfig = payload.request; + const datasetConfig = payload.request; const connectors = _.get(datasetConfig, "connectors_config", []); const transformations = _.get(datasetConfig, "transformations_config", []); @@ -99,7 +99,7 @@ export const migrateExportedDatasetV1 = (requestPayload: Record) => const { dataset_id, timestamp_key = "", data_key = "", type: datasetType } = _.get(datasetPayload, "data.metadata") const type = datasetType === "master-dataset" ? DatasetType.master : DatasetType.event - let dataset: Record = { + const dataset: Record = { dataset_id, id: dataset_id, name: dataset_id, type, version_key: Date.now().toString(), api_version: "v2", diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 28909ff3..fc16b313 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -17,7 +17,7 @@ const validateRequest = (req: Request) => { const { dataset_id } = req.params; const fields = req.query.fields; - if (fields && typeof fields !== 'string') { + if (fields && typeof fields !== "string") { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); } const fieldValues = fields ? _.split(fields, ",") : []; diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 05e42885..8506afd3 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -92,10 +92,10 @@ const deleteDataset = async (dataset: Record) => { const readyForPublish = async (dataset: Record) => { - let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) - if (draftDataset?.type === 'master') { + if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } _.mergeWith(draftDataset,defaultConfigs,draftDataset, (objValue, srcValue) => { @@ -159,9 +159,9 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } if(!_.isEmpty(md)){ if(md.api_version === "v2") - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); else - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); } return datasetStatus; @@ -187,8 +187,8 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } const updateMasterDataConfig = async (draftDataset: Record) => { - if (draftDataset.type === 'master') { - let dataset_config = _.get(draftDataset, "dataset_config") + if (draftDataset.type === "master") { + const dataset_config = _.get(draftDataset, "dataset_config") const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } if (draftDataset.dataset_config.cache_config.redis_db === 0) { diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 93c5a875..f6db4fd4 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -4,7 +4,7 @@ import httpStatus from "http-status"; import createError from "http-errors"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { publishNotificationChannel, testNotificationChannel, updateNotificationChannel } from "../../services/managers"; -import _ from 'lodash'; +import _ from "lodash"; import { updateTelemetryAuditEvent } from "../../services/telemetry"; const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; @@ -16,7 +16,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -29,13 +29,13 @@ const updateHandler = async (request: Request, response: Response, next: NextFun const notificationPayload = notificationPayloadModel?.toJSON(); if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); - if (_.get(notificationPayload, 'status') === "live") { + if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -44,10 +44,10 @@ const listHandler = async (request: Request, response: Response, next: NextFunct try { const { limit, filters, offset } = request.body?.request || {}; const notifications = await Notification.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); - const count = _.get(notifications, 'length'); + const count = _.get(notifications, "length"); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { notifications, ...(count && { count }) } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -61,7 +61,7 @@ const fetchHandler = async (request: Request, response: Response, next: NextFunc updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: notificationPayloadModel?.toJSON() }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -77,7 +77,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun await Notification.update({ status: "retired" }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -94,7 +94,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu Notification.update({ status: "live" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -114,7 +114,7 @@ const testNotifationChannelHandler = async (request: Request, response: Response } ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "Notification Sent" } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3bac60b7..1713d4fa 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -57,7 +57,7 @@ const ResponseHandler = { }, goneResponse: (req: Request, res: Response) => { - const { id, entity } = req as any; + const { id } = req as any; res.status(httpStatus.GONE).json({ id: id, ver: "v1", ts: Date.now(), params: { status: "FAILED", errmsg: "v1 APIs have been replace by /v2 APIs. Please refer to this link for more information" }, responseCode: httpStatus["410_NAME"] }) } } diff --git a/api-service/src/models/Alert.ts b/api-service/src/models/Alert.ts index 7f98cde5..e52952b8 100644 --- a/api-service/src/models/Alert.ts +++ b/api-service/src/models/Alert.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 as uuidv4 } from 'uuid'; +import { v4 as uuidv4 } from "uuid"; export const Alert = sequelize.define("alerts", { id: { diff --git a/api-service/src/models/Metric.ts b/api-service/src/models/Metric.ts index 605ef291..ecc4cb27 100644 --- a/api-service/src/models/Metric.ts +++ b/api-service/src/models/Metric.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 as uuidv4 } from 'uuid'; +import { v4 as uuidv4 } from "uuid"; export const Metrics = sequelize.define("metrics", { id: { diff --git a/api-service/src/models/Notification.ts b/api-service/src/models/Notification.ts index dc2bee5a..6203032c 100644 --- a/api-service/src/models/Notification.ts +++ b/api-service/src/models/Notification.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 } from 'uuid'; +import { v4 } from "uuid"; export const Notification = sequelize.define("notificationchannel", { id: { diff --git a/api-service/src/models/Silence.ts b/api-service/src/models/Silence.ts index 3c272db1..d78d3f89 100644 --- a/api-service/src/models/Silence.ts +++ b/api-service/src/models/Silence.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 as uuid } from 'uuid'; +import { v4 as uuid } from "uuid"; export const Silence = sequelize.define("silences", { id: { diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts index 9efab170..01c843dd 100644 --- a/api-service/src/routes/AlertsRouter.ts +++ b/api-service/src/routes/AlertsRouter.ts @@ -1,5 +1,5 @@ import express from "express"; -import notificationHandler from '../controllers/NotificationChannel/Notification'; +import notificationHandler from "../controllers/NotificationChannel/Notification"; import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; import customAlertHandler from "../controllers/Alerts/Alerts"; import metricAliasHandler from "../controllers/Alerts/Metric"; diff --git a/api-service/src/services/DatasetHealthService.ts b/api-service/src/services/DatasetHealthService.ts index 92672c47..5ea8b6df 100644 --- a/api-service/src/services/DatasetHealthService.ts +++ b/api-service/src/services/DatasetHealthService.ts @@ -19,9 +19,9 @@ const prometheusQueries = { dedupFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_dedup_failed_count[1d]))", denormFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_DATASETID_denorm_failed[1d]))", transformationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_DATASETID_transform_failed_count[1d]))", - queriesCount: 'sum(sum_over_time(node_total_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))', - avgQueryResponseTimeInSec: 'avg(avg_over_time(node_query_response_time{entity="data-out", dataset_id="DATASETID"}[1d]))/1000', - queriesFailedCount: 'sum(sum_over_time(node_failed_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))' + queriesCount: "sum(sum_over_time(node_total_api_calls{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))", + avgQueryResponseTimeInSec: "avg(avg_over_time(node_query_response_time{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))/1000", + queriesFailedCount: "sum(sum_over_time(node_failed_api_calls{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))" } export const getDatasetHealth = async (categories: any, dataset: any) => { diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 92db6058..8c6c1e62 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -115,7 +115,7 @@ class DatasetService { migrateDatasetV1 = async (dataset_id: string, dataset: Record): Promise => { const status = _.get(dataset, "status") - let draftDataset: Record = { + const draftDataset: Record = { api_version: "v2", version_key: Date.now().toString() } @@ -337,7 +337,7 @@ class DatasetService { const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -357,7 +357,7 @@ class DatasetService { export const getLiveDatasetConfigs = async (dataset_id: string) => { - let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) + const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) diff --git a/api-service/src/services/HealthService.ts b/api-service/src/services/HealthService.ts index 273ceb00..c10a9ea6 100644 --- a/api-service/src/services/HealthService.ts +++ b/api-service/src/services/HealthService.ts @@ -1,9 +1,9 @@ -import { Request, Response, NextFunction } from "express" +import { Request, Response } from "express" import { ResponseHandler } from "../helpers/ResponseHandler" class HealthService { - async checkDruidHealth(req: Request, res: Response, next: NextFunction) { + async checkDruidHealth(req: Request, res: Response) { ResponseHandler.successResponse(req, res, { status: 200, data: {} }) } diff --git a/api-service/src/services/WrapperService.ts b/api-service/src/services/WrapperService.ts index 061b2b14..c38afb6d 100644 --- a/api-service/src/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -1,6 +1,5 @@ import axios from "axios"; import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; import { config } from "../configs/Config"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; diff --git a/api-service/src/services/fs.ts b/api-service/src/services/fs.ts index e08597d6..774b87b9 100644 --- a/api-service/src/services/fs.ts +++ b/api-service/src/services/fs.ts @@ -1,8 +1,8 @@ -import fs from 'fs'; -import path from 'path'; +import fs from "fs"; +import path from "path"; export const scrapModules = (folderPath: string, basename: string) => { - const mapping = new Map>(); + const mapping = new Map>(); fs.readdirSync(folderPath) .filter((file) => file !== basename) .map((file) => { diff --git a/api-service/src/services/managers/grafana/alert/helpers/index.ts b/api-service/src/services/managers/grafana/alert/helpers/index.ts index e9c19e29..9c577a52 100644 --- a/api-service/src/services/managers/grafana/alert/helpers/index.ts +++ b/api-service/src/services/managers/grafana/alert/helpers/index.ts @@ -32,7 +32,7 @@ const deleteAlertFolder = async (folderName: string) => { const checkIfGroupNameExists = async (category: string) => { const response = await getRules(); - const rules = _.get(response, 'data'); + const rules = _.get(response, "data"); if(!_.has(rules, category)) return undefined; return _.find(_.flatMap(_.values(rules)), { name: category, @@ -59,7 +59,7 @@ const getSpecificRule = async (payload: Record) => { const alertrules = await alerts(); const groups = _.get(alertrules, "data.data.groups"); const ruleGroup = _.find(groups, (group: any) => group.name == payload.category); - return _.find(_.get(ruleGroup, 'rules'), (rule: any) => rule.name == payload.name); + return _.find(_.get(ruleGroup, "rules"), (rule: any) => rule.name == payload.name); }; const updateMetadata = (metadata: any, dataSource: string, expression: string) => { @@ -89,7 +89,7 @@ const createFolder = (title: string) => { const createFolderIfNotExists = async (folderName: string) => { const folders = await getFolders(); - const isExists = _.find(folders.data, folder => _.get(folder, 'title') === folderName); + const isExists = _.find(folders.data, folder => _.get(folder, "title") === folderName); if (isExists) return; return createFolder(folderName); } @@ -215,7 +215,7 @@ const queryOperators = [ const getQueryExpression = (payload: Record) => { const { metric, operator, threshold } = payload; - const operatorSymbol = _.get(_.find(queryOperators, operatorMetadata => _.get(operatorMetadata, 'value') === operator), 'symbol'); + const operatorSymbol = _.get(_.find(queryOperators, operatorMetadata => _.get(operatorMetadata, "value") === operator), "symbol"); return `(${metric}) ${operatorSymbol} ${threshold}`; } @@ -229,7 +229,7 @@ const getMatchingLabels = async (channels: string[]) => { const { name, type } = channelMetadata; return `notificationChannel_${name}_${type}`; }) - .catch(err => null); + .catch(() => null); } const matchingLabels = await Promise.all(channels.map(fetchChannel)); @@ -245,22 +245,22 @@ const getMatchingLabels = async (channels: string[]) => { const transformRule = async ({ value, condition, metadata, isGroup }: any) => { const { name, id, interval, category, frequency, labels = {}, annotations = {}, severity, description, notification = {} } = value; const annotationObj = { ...annotations, description: description }; - const channels = _.get(notification, 'channels') || []; + const channels = _.get(notification, "channels") || []; const matchingLabelsForNotification = await getMatchingLabels(channels); const payload = { grafana_alert: { title: name, condition: condition, - no_data_state: _.get(metadata, 'no_data_state', 'NoData'), - exec_err_state: _.get(metadata, 'exec_err_state', 'Error'), + no_data_state: _.get(metadata, "no_data_state", "NoData"), + exec_err_state: _.get(metadata, "exec_err_state", "Error"), data: metadata, is_paused: false, }, for: interval, annotations: annotationObj, labels: { - 'alertId': id, + "alertId": id, ...labels, ...(severity && { severity }), ...matchingLabelsForNotification diff --git a/api-service/src/services/managers/grafana/alert/index.ts b/api-service/src/services/managers/grafana/alert/index.ts index b2dcd1e2..471a104d 100644 --- a/api-service/src/services/managers/grafana/alert/index.ts +++ b/api-service/src/services/managers/grafana/alert/index.ts @@ -1,5 +1,5 @@ import _ from "lodash"; -import { addGrafanaRule, checkIfGroupNameExists, checkIfRuleExists, createFolderIfNotExists, deleteAlertFolder, deleteAlertRule, getPrometheusDataSource, getQueryExpression, getQueryModel, getSpecificRule, transformRule, updateMetadata, getFilteredAlerts, getRules, groupRulesByCategory } from "./helpers"; +import { addGrafanaRule, checkIfGroupNameExists, checkIfRuleExists, createFolderIfNotExists, deleteAlertFolder, deleteAlertRule, getPrometheusDataSource, getQueryExpression, getQueryModel, getSpecificRule, transformRule, updateMetadata, getRules } from "./helpers"; import { Silence } from "../../../../models/Silence"; import constants from "../../constants"; @@ -27,7 +27,7 @@ const publishAlert = async (payload: Record) => { const getAlerts = async (payload: Record) => { const context = payload?.context || {}; - const alertId = _.get(payload, 'id'); + const alertId = _.get(payload, "id"); const { err, alertData } = await getSpecificRule(payload) .then(alertData => { if (!alertData) throw new Error() @@ -37,21 +37,21 @@ const getAlerts = async (payload: Record) => { const silenceModel = await Silence.findOne({ where: { alert_id: alertId } }); const silenceData = silenceModel?.toJSON(); - let silenceState: Record = { state: '', silenceId: '' }; + const silenceState: Record = { state: "", silenceId: "" }; if (silenceData) { const { end_time } = silenceData; const currentTime = new Date().getTime(); const endTime = new Date(end_time).getTime(); if (currentTime < endTime) { - silenceState.state = 'muted'; - silenceState['endTime'] = endTime; + silenceState.state = "muted"; + silenceState["endTime"] = endTime; } else { - silenceState.state = 'unmuted'; + silenceState.state = "unmuted"; } silenceState.silenceId = silenceData.id; } else { - silenceState.state = 'unmuted'; + silenceState.state = "unmuted"; } return { ...payload, context: { ...context, err }, ...(alertData && { alertData }), silenceState }; @@ -62,8 +62,8 @@ const deleteAlert = async (payload: Record) => { const alertCategory = await checkIfGroupNameExists(category); if (!alertCategory) throw new Error(constants.CATEGORY_NOT_EXIST); - if (_.get(alertCategory, 'rules.length') > 1) { - const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, 'grafana_alert.title') !== name) || []; + if (_.get(alertCategory, "rules.length") > 1) { + const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, "grafana_alert.title") !== name) || []; const filteredGroup = { ...alertCategory, rules: filteredRule }; return addGrafanaRule(filteredGroup); } @@ -83,19 +83,19 @@ const generateAlertPayload = (payload: Record) => { } const filterSystemRulesPredicate = (rule: Record) => { - const labels = _.get(rule, 'labels') || {}; + const labels = _.get(rule, "labels") || {}; const isSystemAlert = _.find(labels, (value, key) => (key === "alertSource" && value === "system-rule-ingestor-job")); if (!isSystemAlert) return true; return false; } -const deleteSystemRules = async (filters: Record) => { +const deleteSystemRules = async () => { const response = await getRules(); - const existingRules = _.cloneDeep(_.get(response, 'data')) as Record[]>; + const existingRules = _.cloneDeep(_.get(response, "data")) as Record[]>; for (const [category, evaluationGroups] of Object.entries(existingRules)) { const evaluationGroup = _.find(evaluationGroups, ["name", category]); if (!evaluationGroup) continue; - const rules = _.get(evaluationGroup, 'rules') || [] + const rules = _.get(evaluationGroup, "rules") || [] const filteredRules = _.filter(rules, filterSystemRulesPredicate); try { if (_.isEmpty(filteredRules)) { diff --git a/api-service/src/services/managers/grafana/index.ts b/api-service/src/services/managers/grafana/index.ts index 30c2b82d..1824c6c5 100644 --- a/api-service/src/services/managers/grafana/index.ts +++ b/api-service/src/services/managers/grafana/index.ts @@ -1,6 +1,6 @@ -import * as alertFunctions from './alert' -import * as notificationFunctions from './notification'; -import * as silenceFunctions from './silences'; +import * as alertFunctions from "./alert" +import * as notificationFunctions from "./notification"; +import * as silenceFunctions from "./silences"; const service = { name: "grafana", ...alertFunctions, ...notificationFunctions, ...silenceFunctions }; export default service \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/email.ts b/api-service/src/services/managers/grafana/notification/channels/email.ts index d4628517..06658016 100644 --- a/api-service/src/services/managers/grafana/notification/channels/email.ts +++ b/api-service/src/services/managers/grafana/notification/channels/email.ts @@ -1,6 +1,6 @@ -import _ from 'lodash'; +import _ from "lodash"; import { IChannelConfig } from "../../../../../types/AlertModels"; -import { grafanaHttpClient } from '../../../../../connections/grafanaConnection'; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; const getReceiverObject = ({ name, recipientAddresses, message, multipleAddresses, type }: any) => { return { @@ -30,7 +30,7 @@ const service: IChannelConfig = { generateConfigPayload(payload: Record): Record { const { type, config, name } = payload; const { recipientAddresses, message, subject = "Obsrv Alert", labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; - const multipleAddresses = _.size(_.split(recipientAddresses, ';')) > 1; + const multipleAddresses = _.size(_.split(recipientAddresses, ";")) > 1; return { notificationPolicy: { receiver: name, @@ -42,7 +42,7 @@ const service: IChannelConfig = { testChannel(payload: Record): Promise { const { name, type, config, message = "Test Channel" } = payload; const { recipientAddresses, subject = "Obsrv Alert" } = config; - const multipleAddresses = _.size(_.split(recipientAddresses, ';')) > 1; + const multipleAddresses = _.size(_.split(recipientAddresses, ";")) > 1; const alert = { annotations: { description: message }, labels: {} }; const body = { alert, receivers: [getReceiverObject({ name, type, multipleAddresses, recipientAddresses, subject })] }; return grafanaHttpClient.post("api/alertmanager/grafana/config/api/v1/receivers/test", body); diff --git a/api-service/src/services/managers/grafana/notification/channels/index.ts b/api-service/src/services/managers/grafana/notification/channels/index.ts index e7f5f87f..abc6b45a 100644 --- a/api-service/src/services/managers/grafana/notification/channels/index.ts +++ b/api-service/src/services/managers/grafana/notification/channels/index.ts @@ -1,11 +1,11 @@ -import path from 'path'; -import { scrapModules } from '../../../../../services/fs'; -import { IChannelConfig } from '../../../../../types/AlertModels'; +import path from "path"; +import { scrapModules } from "../../../../../services/fs"; +import { IChannelConfig } from "../../../../../types/AlertModels"; const channels = scrapModules(__dirname, path.basename(__filename)); export const getChannelService = (channelName: string) => { const channel = channels.get(channelName.toLowerCase()); - if (!channel) throw new Error('invalid channel'); + if (!channel) throw new Error("invalid channel"); return channel; } \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/slack.ts b/api-service/src/services/managers/grafana/notification/channels/slack.ts index 2da034fc..983d9879 100644 --- a/api-service/src/services/managers/grafana/notification/channels/slack.ts +++ b/api-service/src/services/managers/grafana/notification/channels/slack.ts @@ -1,7 +1,6 @@ -import axios from 'axios'; -import _ from 'lodash'; -import CONSTANTS from '../../../constants' -import { IChannelConfig } from '../../../../../types/AlertModels'; +import axios from "axios"; +import CONSTANTS from "../../../constants" +import { IChannelConfig } from "../../../../../types/AlertModels"; const generateConfigPayload = (payload: Record): Record => { const { type, config, name } = payload; diff --git a/api-service/src/services/managers/grafana/notification/channels/teams.ts b/api-service/src/services/managers/grafana/notification/channels/teams.ts index 5cc0e689..714f4f19 100644 --- a/api-service/src/services/managers/grafana/notification/channels/teams.ts +++ b/api-service/src/services/managers/grafana/notification/channels/teams.ts @@ -1,7 +1,6 @@ -import axios from 'axios'; -import _ from 'lodash'; -import CONSTANTS from '../../../constants'; -import { IChannelConfig } from '../../../../../types/AlertModels'; +import axios from "axios"; +import CONSTANTS from "../../../constants"; +import { IChannelConfig } from "../../../../../types/AlertModels"; const generateConfigPayload = (payload: Record): Record => { const {type, config, name} = payload; diff --git a/api-service/src/services/managers/grafana/notification/helpers/index.ts b/api-service/src/services/managers/grafana/notification/helpers/index.ts index a90c2ad7..112365b7 100644 --- a/api-service/src/services/managers/grafana/notification/helpers/index.ts +++ b/api-service/src/services/managers/grafana/notification/helpers/index.ts @@ -1,7 +1,7 @@ import _ from "lodash"; import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; -import { getChannelService } from '../channels'; -import defaultTemplates from '../templates'; +import { getChannelService } from "../channels"; +import defaultTemplates from "../templates"; const generateChannelConfig = (payload: Record) => { const { type } = payload; @@ -18,9 +18,9 @@ const updateAlertManagerConfig = async (payload: Record) => { return grafanaHttpClient.post("/api/alertmanager/grafana/config/api/v1/alerts", payload); }; -const getReceivers = (alertmanager_config: Record) => _.get(alertmanager_config, 'alertmanager_config.receivers') as Array; -const getRoutes = (alertmanager_config: Record) => _.get(alertmanager_config, 'alertmanager_config.route.routes') as Array; -const getTemplates = (alertmanager_config: Record) => _.get(alertmanager_config, 'template_files') as Record; +const getReceivers = (alertmanager_config: Record) => _.get(alertmanager_config, "alertmanager_config.receivers") as Array; +const getRoutes = (alertmanager_config: Record) => _.get(alertmanager_config, "alertmanager_config.route.routes") as Array; +const getTemplates = (alertmanager_config: Record) => _.get(alertmanager_config, "template_files") as Record; const createContactPointsAndNotificationPolicy = async (metadata: Record) => { const { receiver, notificationPolicy } = metadata; @@ -28,9 +28,9 @@ const createContactPointsAndNotificationPolicy = async (metadata: Record, alert const clonedAlertManagerConfig = _.cloneDeep(alertManagerConfig); const existingReceivers = getReceivers(clonedAlertManagerConfig) || []; const existingRoutes = getRoutes(clonedAlertManagerConfig) || []; - _.remove(existingRoutes, route => _.get(route, 'receiver') === name); - _.remove(existingReceivers, receiver => _.get(receiver, 'name') === name); + _.remove(existingRoutes, route => _.get(route, "receiver") === name); + _.remove(existingReceivers, receiver => _.get(receiver, "name") === name); return clonedAlertManagerConfig; }; diff --git a/api-service/src/services/managers/grafana/notification/index.ts b/api-service/src/services/managers/grafana/notification/index.ts index bf7f387b..816e473c 100644 --- a/api-service/src/services/managers/grafana/notification/index.ts +++ b/api-service/src/services/managers/grafana/notification/index.ts @@ -1,5 +1,4 @@ -import _ from "lodash"; -import { getChannelService } from './channels'; +import { getChannelService } from "./channels"; import { createContactPointsAndNotificationPolicy, generateChannelConfig, getAlertManagerConfig, removeReceiverAndNotificationPolicy, updateAlertManagerConfig } from "./helpers"; const updateNotificationChannel = async (payload: Record) => { diff --git a/api-service/src/services/managers/grafana/silences/helpers/index.ts b/api-service/src/services/managers/grafana/silences/helpers/index.ts index 599ab05f..50af380c 100644 --- a/api-service/src/services/managers/grafana/silences/helpers/index.ts +++ b/api-service/src/services/managers/grafana/silences/helpers/index.ts @@ -19,7 +19,7 @@ const disableSilence = (silenceId: string) => { const getCurrentSilenceStatus = async (silenceId: string) => { const response = await getSilence(silenceId); - const currentSilenceStatus = _.get(response, 'data.status.state'); + const currentSilenceStatus = _.get(response, "data.status.state"); return currentSilenceStatus; } diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index e2e57568..84b666d9 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -61,9 +61,9 @@ export const deleteAlertRule = async (payload: Record, hardDelete: export const deleteSystemRules = async (payload: Record) => { - const { rules = [], manager } = payload; + const { manager } = payload; const service = getService(manager); - return service.deleteSystemRules(rules); + return service.deleteSystemRules(); } export const getAlertsMetadata = (payload: Record) => { @@ -125,7 +125,7 @@ export const deleteAlertByDataset = async (payload: Record) => { const { name } = payload; const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) - for (let payload of alertRulePayload) { + for (const payload of alertRulePayload) { await deleteAlertRule(payload, true) await retireAlertSilence(_.get(payload, "id") || "") } @@ -140,7 +140,7 @@ export const deleteMetricAliasByDataset = async (payload: Record) = const { name } = payload; const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) - for (let payload of metricAliasPayload) { + for (const payload of metricAliasPayload) { await payload.destroy() } return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; @@ -173,7 +173,7 @@ export const getAlertMetricsByDataset = async (payload: Record) => export const createAlertsByDataset = async (payload: any) => { try { - for (let alerts of payload) { + for (const alerts of payload) { const alertPayload = _.omit(alerts as any, ["id", "status", "createdAt", "updatedAt", "created_by", "updated_by"]) await Alert.create(alertPayload) } @@ -184,7 +184,7 @@ export const createAlertsByDataset = async (payload: any) => { export const createMetricAliasByDataset = async (payload: any) => { try { - for (let metrics of payload) { + for (const metrics of payload) { const metricsPayload = _.omit(metrics as any, ["id", "createdAt", "updatedAt"]) await Metrics.create(metricsPayload) } @@ -198,7 +198,7 @@ export const publishAlertByDataset = async (payload: Record) => { const { name } = payload; const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) if (!alertRulePayload) throw new Error("Alert rule does not exist") - for (let payload of alertRulePayload) { + for (const payload of alertRulePayload) { await publishAlert(payload) } return constants.ALERTS_PUBLISHED_SUCCESSFULLY; diff --git a/api-service/src/services/managers/prometheus/alert/index.ts b/api-service/src/services/managers/prometheus/alert/index.ts index f2bd3a2f..28f5412f 100644 --- a/api-service/src/services/managers/prometheus/alert/index.ts +++ b/api-service/src/services/managers/prometheus/alert/index.ts @@ -1,17 +1,17 @@ -import CONSTANTS from '../../constants' +import CONSTANTS from "../../constants" -export const publishAlert = async (payload: Record) => { +export const publishAlert = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const getAlerts = async (payload: Record) => { +export const getAlerts = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const deleteAlert = async (payload: Record) => { +export const deleteAlert = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const generateAlertPayload = (payload: Record) => { +export const generateAlertPayload = () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const deleteSystemRules= async (payload: Record) => { +export const deleteSystemRules= async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } diff --git a/api-service/src/services/managers/prometheus/index.ts b/api-service/src/services/managers/prometheus/index.ts index c8c2e7df..f8f0da66 100644 --- a/api-service/src/services/managers/prometheus/index.ts +++ b/api-service/src/services/managers/prometheus/index.ts @@ -1,5 +1,5 @@ -import * as alertFunctions from './alert' -import * as notificationFunctions from './notification'; -import * as silenceFunctions from './silences'; +import * as alertFunctions from "./alert" +import * as notificationFunctions from "./notification"; +import * as silenceFunctions from "./silences"; export default { ...alertFunctions, ...notificationFunctions, ...silenceFunctions } \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/notification/index.ts b/api-service/src/services/managers/prometheus/notification/index.ts index 91b663cb..9514d324 100644 --- a/api-service/src/services/managers/prometheus/notification/index.ts +++ b/api-service/src/services/managers/prometheus/notification/index.ts @@ -1,11 +1,11 @@ -import CONSTANTS from '../../constants' +import CONSTANTS from "../../constants" -export const createNotificationChannel = (payload: Record) => { +export const createNotificationChannel = () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -export const testNotificationChannel = async (payload: Record) => { +export const testNotificationChannel = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -export const updateNotificationChannel = (payload: Record) => { +export const updateNotificationChannel = () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/silences/index.ts b/api-service/src/services/managers/prometheus/silences/index.ts index 1c073881..60ed0aef 100644 --- a/api-service/src/services/managers/prometheus/silences/index.ts +++ b/api-service/src/services/managers/prometheus/silences/index.ts @@ -1,18 +1,18 @@ -import CONSTANTS from '../../constants'; +import CONSTANTS from "../../constants"; -const createSilence = async (payload: Record) => { +const createSilence = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -const getSilenceMetadata = async (payload: Record) => { +const getSilenceMetadata = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -const updateSilence = async (silence: Record, payload: Record) => { +const updateSilence = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -const deleteSilence = async (payload: Record) => { +const deleteSilence = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } diff --git a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts index a6037fff..487cd77e 100644 --- a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import { TestInputsForDataIngestion } from "./Fixtures"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { Dataset } from "../../../models/Dataset"; import sinon from "sinon"; import { Kafka } from "kafkajs"; @@ -14,17 +14,16 @@ chai.should(); chai.use(chaiHttp); const kafka = new Kafka(connectionConfig.kafka.config); -const producer = kafka.producer(); const apiEndpoint = "/v2/data/in/:datasetId" const resultResponse = [ { - topicName: 'local.test.topic', + topicName: "local.test.topic", partition: 0, errorCode: 0, - baseOffset: '257', - logAppendTime: '-1', - logStartOffset: '0' + baseOffset: "257", + logAppendTime: "-1", + logStartOffset: "0" } ] const kafkaModule = require("../../../connections/kafkaConnection"); @@ -39,7 +38,7 @@ describe("DATA INGEST API", () => { return Promise.resolve({ dataValues: { dataset_config: { - entry_topic: 'local.test.topic', + entry_topic: "local.test.topic", }, extraction_config: { is_batch_event: false, @@ -74,7 +73,7 @@ describe("DATA INGEST API", () => { return Promise.resolve({ dataValues: { dataset_config: { - entry_topic: 'local.test.topic', + entry_topic: "local.test.topic", } } }) @@ -103,7 +102,7 @@ describe("DATA INGEST API", () => { return Promise.resolve({ dataValues: { dataset_config: { - entry_topic: 'local.test.topic', + entry_topic: "local.test.topic", } } }) diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 3a7be3dc..96d53e38 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import nock from "nock"; import { TestQueries } from "./Fixtures"; import { config } from "../../../configs/Config"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; +import chaiSpies from "chai-spies" +import { describe, it } from "mocha"; import { Datasource } from "../../../models/Datasource"; chai.use(chaiSpies) chai.should(); @@ -35,7 +35,7 @@ describe("QUERY API TESTS", () => { }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) - .reply(200, ['telemetry-events.1_rollup']) + .reply(200, ["telemetry-events.1_rollup"]) chai .request(app) .post("/v2/data/query/telemetry-events") diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts index 8f9bd9f7..c11de5f1 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts @@ -1,23 +1,23 @@ export const TestQueries = { VALID_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"test","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2024-01-31/2024-02-01"]},"granularity":"week","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}]}}', - HIGH_LIMIT_NATIVE_QUERY: '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2024-11-31/2024-12-01"]},"granularity":"day","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}],"limit":10000,"threshold":10000}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"test\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2024-01-31/2024-02-01\"]},\"granularity\":\"week\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}]}}", + HIGH_LIMIT_NATIVE_QUERY: "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2024-11-31/2024-12-01\"]},\"granularity\":\"day\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}],\"limit\":10000,\"threshold\":10000}}", WITHOUT_THRESOLD_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events"},"query":{"queryType":"timeBoundary","dimension":"content_status","metric":"count","granularity":"all","intervals":["2020-12-21/2020-12-22"],"aggregations":[]}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\"},\"query\":{\"queryType\":\"timeBoundary\",\"dimension\":\"content_status\",\"metric\":\"count\",\"granularity\":\"all\",\"intervals\":[\"2020-12-21/2020-12-22\"],\"aggregations\":[]}}", VALID_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"test","aggregationLevel":"week"},"query":"SELECT * FROM \\"test\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"test\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21' LIMIT 10\"}", HIGH_LIMIT_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events"},"querySql":{"query":"SELECT msgid FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-22\' LIMIT 100000"}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\"},\"querySql\":{\"query\":\"SELECT msgid FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2021-01-22' LIMIT 100000\"}}", HIGH_DATE_RANGE_SQL_QUERY: `{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2022-02-12' LIMIT 10"}`, LIMIT_IS_NAN: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"dataSource":"telemetry-events","aggregationLevel":"week"},"query":"SELECT content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-12\' LIMIT 100"}', - DATASOURCE_NOT_FOUND: '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry"},"query":"SELECT content_status FROM \\"telemetry\\" LIMIT 5"}', - INVALID_DATE_RANGE_NATIVE: '{"id":"api.data.out","ver":"1.0","ts":"1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","table":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2023-01-31/2023-04-01"]},"granularity":"day","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"}]}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"dataSource\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT content_status FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2021-01-12' LIMIT 100\"}", + DATASOURCE_NOT_FOUND: "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\"},\"query\":\"SELECT content_status FROM \\\"telemetry\\\" LIMIT 5\"}", + INVALID_DATE_RANGE_NATIVE: "{\"id\":\"api.data.out\",\"ver\":\"1.0\",\"ts\":\"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"table\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2023-01-31/2023-04-01\"]},\"granularity\":\"day\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"}]}}", INVALID_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry","aggregationLevel":"week"},"query":"SELECT * FROM \\"telemetry\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"telemetry\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21' LIMIT 10\"}", VALID_SQL_QUERY_WITHOUT_LIMIT: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry","aggregationLevel":"week"},"query":"SELECT * FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\'"}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21'\"}", VALID_INTERVAL: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":"2024-01-31/2024-02-01","granularity":"week","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}]}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":\"2024-01-31/2024-02-01\",\"granularity\":\"week\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}]}}", } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 10cd9418..a0e2ace6 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -4,10 +4,9 @@ import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; import { TestInputsForDatasetCreate, DATASET_CREATE_SUCCESS_FIXTURES, DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES } from "./Fixtures"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { sequelize } from "../../../connections/databaseConnection"; -import _ from "lodash"; import { apiId } from "../../../controllers/DatasetCreate/DatasetCreate" import { Dataset } from "../../../models/Dataset"; diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts index b3138c9f..46aaebeb 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -1,5 +1,4 @@ import httpStatus from "http-status" -import _ from "lodash" export const TestInputsForDatasetCreate = { VALID_DATASET: { diff --git a/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts index 8b55ee75..380ee8ad 100644 --- a/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts @@ -1,15 +1,14 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId } from "../../../controllers/DatasetList/DatasetList"; import { TestInputsForDatasetList } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; -import { DatasetTransformations } from "../../../models/Transformation"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 6accc269..8b104ec9 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; import { TestInputsForDatasetRead } from "./Fixtures"; @@ -25,7 +25,7 @@ describe("DATASET READ API", () => { it("Dataset read success: When minimal fields requested", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ 'name': 'sb-telemetry', 'data_version': 1 }) + return Promise.resolve({ "name": "sb-telemetry", "data_version": 1 }) }) chai .request(app) @@ -36,9 +36,9 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') + res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: 'sb-telemetry', data_version: 1 })) + result.should.be.eq(JSON.stringify({ name: "sb-telemetry", data_version: 1 })) done(); }); }); @@ -56,8 +56,8 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.type.should.be.eq('event') - res.body.result.status.should.be.eq('Draft') + res.body.result.type.should.be.eq("event") + res.body.result.status.should.be.eq("Draft") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA })) done(); @@ -77,7 +77,7 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.status.should.be.eq('Live') + res.body.result.status.should.be.eq("Live") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) done(); @@ -109,7 +109,7 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') + res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); @@ -141,7 +141,7 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') + res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts index f3de6b4f..0016ae5d 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 7c379727..29edc291 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 8d83d915..388a224d 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 0e213eb2..c5c32929 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts index da2d4450..fa394a8a 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index 9f1d5258..e8f614c8 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index 482ed218..cdf02c4b 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -3,7 +3,7 @@ import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 964a420f..24ccc831 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts index cea56f57..fe3715d4 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -3,7 +3,7 @@ import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index 96da9863..0ae345ff 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -3,7 +3,7 @@ import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index bec2db18..1818bc79 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -3,7 +3,7 @@ import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index c69d3de1..e77ab8b0 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts index ee904424..06dccdaf 100644 --- a/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts @@ -55,21 +55,21 @@ export const TestInputsForGenerateURL = { VALID_RESPONSE_FOR_MULTIFILES: [ { "filePath": `container/api-service/user-upload/telemetry.json`, - "fileName": 'telemetry.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC' + "fileName": "telemetry.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC" }, { - "filePath": 'container/api-service/user-upload/school-data.json', - "fileName": 'school-data.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/school-data.json?X-Amz-Algorithm=AWS4-HMAC' + "filePath": "container/api-service/user-upload/school-data.json", + "fileName": "school-data.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/school-data.json?X-Amz-Algorithm=AWS4-HMAC" } ], VALID_RESPONSE_FOR_SINGLE_FILE: [ { - "filePath": 'container/api-service/user-upload/telemetry.json', - "fileName": 'telemetry.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC' + "filePath": "container/api-service/user-upload/telemetry.json", + "fileName": "telemetry.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC" } ] } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts index 8d30ddb6..597b3b7c 100644 --- a/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts +++ b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId, code } from "../../../controllers/GenerateSignedURL/GenerateSignedURL"; import { TestInputsForGenerateURL } from "./Fixtures"; diff --git a/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts index b7f60e27..4bedf75f 100644 --- a/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { createTemplateFixtures } from "./Fixtures" import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.create' +const apiId = "api.query.template.create" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts index 4108415b..47377590 100644 --- a/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts @@ -1,10 +1,10 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.delete' +const apiId = "api.query.template.delete" chai.use(spies); chai.should(); @@ -20,7 +20,7 @@ describe("DELETE QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "destroy", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1' + template_id: "sql1" } }) }) diff --git a/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts index e9f843d1..16800dcd 100644 --- a/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts +++ b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; import { listTemplateFixtures } from "./Fixtures"; -const apiId = 'api.query.template.list' +const apiId = "api.query.template.list" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts index 1f6e129e..6a7078f2 100644 --- a/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts @@ -1,10 +1,10 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.read' +const apiId = "api.query.template.read" chai.use(spies); chai.should(); @@ -20,12 +20,12 @@ describe("READ QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1', - template_name: 'sql1', - query: '"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} LIMIT 1"', - query_type: 'sql', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "sql1", + template_name: "sql1", + query: "\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} LIMIT 1\"", + query_type: "sql", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-29T11:29:58.759Z", updated_date: "2024-04-29T11:29:58.759Z" } diff --git a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts index ff384181..8da1818b 100644 --- a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts +++ b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts @@ -1,14 +1,14 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; import { Datasource } from "../../../models/Datasource"; import nock from "nock"; import { config } from "../../../configs/Config"; import { templateQueryApiFixtures } from "./Fixtures"; -const apiId = 'api.query.template.query'; +const apiId = "api.query.template.query"; const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" chai.use(spies); @@ -34,12 +34,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1', - template_name: 'sql1', - query: '"SELECT * FROM {{DATASET}} WHERE \"__time\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}"', - query_type: 'sql', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "sql1", + template_name: "sql1", + query: "\"SELECT * FROM {{DATASET}} WHERE \"__time\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\"", + query_type: "sql", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04 - 30T05: 57:04.387Z", updated_date: "2024-04 - 30T05: 57:04.387Z" } @@ -77,12 +77,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'jsontemplate1', - template_name: 'jsontemplate1', - query: '{"queryType":"timeseries","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMIT}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]}', - query_type: 'json', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "jsontemplate1", + template_name: "jsontemplate1", + query: "{\"queryType\":\"timeseries\",\"datasetId\":\"{{DATASET}}\",\"intervals\":\"{{STARTDATE}}/{{ENDDATE}}\",\"limit\":\"{{LIMIT}}\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"school_id\"}},\"name\":\"school_id\"}]}", + query_type: "json", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-28T23:28:35.868Z", updated_date: "2024-04-28T23:28:35.868Z" } @@ -121,12 +121,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'jsontemplate1', - template_name: 'jsontemplate1', - query: '{"queryType":"timeseries","datasetId"::::"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMIT}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]}', - query_type: 'json', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "jsontemplate1", + template_name: "jsontemplate1", + query: "{\"queryType\":\"timeseries\",\"datasetId\"::::\"{{DATASET}}\",\"intervals\":\"{{STARTDATE}}/{{ENDDATE}}\",\"limit\":\"{{LIMIT}}\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"school_id\"}},\"name\":\"school_id\"}]}", + query_type: "json", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-28T23:28:35.868Z", updated_date: "2024-04-28T23:28:35.868Z" } diff --git a/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts index 195018c3..af50d77a 100644 --- a/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { updateTemplateFixtures } from "./Fixtures" import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.update' +const apiId = "api.query.template.update" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts index bd1e1a31..510b8af4 100644 --- a/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts +++ b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts @@ -1,8 +1,8 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForSqlWrapper } from "./Fixtures"; import httpStatus from "http-status"; From 0adc92de744357918003328cfb3d6760324bbf58 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 5 Aug 2024 14:40:59 +0530 Subject: [PATCH 047/311] #OBS-I116: feat: Dataset status transition test cases fix --- .../DatasetLive.spec.ts | 49 ++- .../DatasetReadyToPublish.spec.ts | 10 +- .../DatasetStatusTransition.spec.ts | 13 +- .../DatasetStatusTransition/Fixtures.ts | 307 ++++++++---------- 4 files changed, 177 insertions(+), 202 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 29edc291..bb0c8e51 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -9,6 +9,8 @@ import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { commandHttpService } from "../../../connections/commandServiceConnection"; import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { Dataset } from "../../../models/Dataset"; chai.use(spies); chai.should(); @@ -24,9 +26,18 @@ describe("DATASET STATUS TRANSITION LIVE", () => { it("Dataset status transition success: When the action is to set dataset live", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "ReadyToPublish" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH) }) - chai.spy.on(commandHttpService, "post", () => { + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -35,6 +46,9 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(t, "commit", () => { return Promise.resolve({}) }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) chai .request(app) .post("/v2/datasets/status-transition") @@ -47,7 +61,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) res.body.result.message.should.be.eq("Dataset status transition to Live successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") + res.body.result.dataset_id.should.be.eq("telemetry") done(); }); }); @@ -66,7 +80,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }) @@ -74,17 +88,29 @@ describe("DATASET STATUS TRANSITION LIVE", () => { it("Dataset status transition failure: When the command api call to publish dataset fails", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "ReadyToPublish" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH) }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.reject() + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) - chai.spy.on(t, "rollback", () => { + chai.spy.on(t, "commit", () => { return Promise.resolve({}) }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.reject() + }) chai .request(app) .post("/v2/datasets/status-transition") @@ -94,26 +120,25 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") done(); }); }); it("Dataset status transition failure: When the dataset to publish is in draft state", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Draft" }) + return Promise.resolve({...TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH, status: "Draft"}) }) chai .request(app) .post("/v2/datasets/status-transition") .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); + res.should.have.status(httpStatus.CONFLICT); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.error.code.should.be.eq("DATASET_LIVE_FAILURE") - res.body.error.message.should.be.eq("Failed to mark dataset Live as it is not in ready to publish state") + res.body.error.message.should.be.eq("Transition failed for dataset: telemetry status:Draft with status transition to Live") done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 388a224d..c82e809d 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -47,7 +47,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") + res.body.result.dataset_id.should.be.eq("telemetry") done(); }); }); @@ -66,7 +66,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); @@ -74,19 +74,19 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { it("Dataset status transition failure: When dataset is already ready to publish", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({dataset_id:"telemetry", status:"ReadyToPublish"}) + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_READY_TO_PUBLISH, "status": "ReadyToPublish" }) }) chai .request(app) .post("/v2/datasets/status-transition") .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); + res.should.have.status(httpStatus.CONFLICT); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to mark dataset Ready to publish as it not in draft state") + res.body.error.message.should.be.eq("Transition failed for dataset: dataset-all-fields7 status:ReadyToPublish with status transition to ReadyToPublish") res.body.error.code.should.be.eq("DATASET_READYTOPUBLISH_FAILURE") done(); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts index fa394a8a..7117aa94 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts @@ -22,7 +22,7 @@ describe("DATASET STATUS TRANSITION API", () => { }); it("Dataset status transition failure: Invalid request payload provided", (done) => { - + chai .request(app) .post("/v2/datasets/status-transition") @@ -39,20 +39,21 @@ describe("DATASET STATUS TRANSITION API", () => { }); }); - it("Dataset status transition failure: Connection to the database failed", (done) => { + it("Dataset status transition failure: When the action is performed on v1 apis", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.reject() + return Promise.resolve({ api_version: "v1" }) }) chai .request(app) .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") + res.body.error.code.should.be.eq("DATASET_API_VERSION_MISMATCH") + res.body.error.message.should.be.eq("Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset") done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index 7deb12f7..db14c924 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -1,187 +1,136 @@ export const TestInputsForDatasetStatusTransition = { - VALID_SCHEMA_FOR_DELETE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Delete" - } + VALID_SCHEMA_FOR_DELETE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_SCHEMA_FOR_LIVE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Live" - } + "request": { + "dataset_id": "telemetry.1", + "status": "Delete" + } + }, + VALID_SCHEMA_FOR_LIVE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_SCHEMA_FOR_RETIRE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry", - "status": "Retire" - } + "request": { + "dataset_id": "telemetry", + "status": "Live" + } + }, + VALID_SCHEMA_FOR_RETIRE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - INVALID_SCHEMA: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "" - } + "request": { + "dataset_id": "telemetry", + "status": "Retire" + } + }, + INVALID_SCHEMA: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_REQUEST_FOR_READY_FOR_PUBLISH:{ - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "ReadyToPublish" + "request": { + "dataset_id": "telemetry.1", + "status": "" + } + }, + VALID_REQUEST_FOR_READY_FOR_PUBLISH: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry", + "status": "ReadyToPublish" + } + }, + VALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "data_key": "eid", "partition_key": "eid", "timestamp_key": "obsrv_meta.syncts" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": [] }, + "tags": ["tag1"], + "status": "Draft", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-07-24 19:12:13.021", + "updated_date": "2024-07-25 06:12:38.412", + "version_key": "1721887933020", + "api_version": "v2", + "transformations_config": [{ "field_key": "email", "transformation_function": { "type": "mask", "expr": "mid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + "connectors_config": [{ "id": "91898e828u82882u8", "connector_id": "kafka", "connector_config": "AR/hz8iBxRyc9s0ohXa3+id+7GoWtjVjNWvurFFgV1Ocw2kgc+XVbnfXX26zkP3+rQ49gio0JzwsFzOK61TtXLx968IKol5eGfaEHF68O5faoxxjKBsyvhPaRQ91DKKi", "version": "v1" }], + "sample_data": {}, + "entry_topic": "local.ingest" + }, + INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "dataset_id": "telemetry", + "type": "", + "name": "sb-telemetry", + "id": "telemetry.1", + "status": "Draft", + "version_key": "1789887878", + "validation_config": { + "validate": true, + "mode": "Strict" + }, + "router_config": { + "topic": "test" + }, + "denorm_config": { + "redis_db_host": "local", + "redis_db_port": 5432, + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "dataset_name": "name", + "dataset_id": "name" + }, + { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_name": "name", + "dataset_id": "name" } + ] }, - VALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "dataset", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "batch_id": "id", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 3783 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 3783 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "ets": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - }, - INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - } + "dataset_config": { + "data_key": "mid", + "timestamp_key": "ets", + "entry_topic": "topic", + "redis_db_host": "local", + "redis_db_port": 5432, + "redis_db": 0, + "index_data": true + }, + "client_state": {}, + "tags": [ + "tag1", + "tag2" + ] + }, + DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From f0e53e149620c2845a99b46ecae0bbb45ff56d4f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:33:06 +0530 Subject: [PATCH 048/311] #OBS-I141: added a new metric to sum the response time --- api-service/src/metrics/prometheus/helpers.ts | 17 +++++++++++++---- api-service/src/metrics/prometheus/index.ts | 7 ++++--- api-service/src/metrics/prometheus/metrics.ts | 10 +++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 5e0442c1..13d7f9d7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -1,5 +1,5 @@ import { NextFunction, Response } from "express"; -import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; +import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; @@ -19,7 +19,10 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -28,7 +31,10 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -37,7 +43,10 @@ export const onGone = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 410 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } diff --git a/api-service/src/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts index da143f9f..3fdefe19 100644 --- a/api-service/src/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -1,6 +1,6 @@ import client from "prom-client"; -import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric } from "./metrics" -const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric]; +import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric } from "./metrics" +const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric]; import { NextFunction } from "express"; const register = new client.Registry(); @@ -17,6 +17,7 @@ const incrementApiCalls = ({ labels = {} }: Record) => totalApiCall const setQueryResponseTime = ({ labels = {}, duration }: Record) => queryResponseTimeMetric.labels(labels).set(duration); const incrementFailedApiCalls = ({ labels = {} }: Record) => failedApiCallsMetric.labels(labels).inc(); const incrementSuccessfulApiCalls = ({ labels = {} }: Record) => successApiCallsMetric.labels(labels).inc(); +const incrementResponseTime = ({ labels = {}, duration }: Record) => sumResponseTimeMetric.labels(labels).inc(duration); //register the metrics configureRegistry(register); @@ -31,5 +32,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls }; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls, incrementResponseTime}; diff --git a/api-service/src/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts index 2985dc0f..499bd15a 100644 --- a/api-service/src/metrics/prometheus/metrics.ts +++ b/api-service/src/metrics/prometheus/metrics.ts @@ -28,9 +28,17 @@ const successApiCallsMetric = new Prometheus.Counter({ labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] }) +// Create a new Prometheus Counter for sum of response time +const sumResponseTimeMetric = new Prometheus.Counter({ + name: "node_sum_response_time", + help: "The sum of response time for time series of same label", + labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] +}); + export { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, - successApiCallsMetric + successApiCallsMetric, + sumResponseTimeMetric } \ No newline at end of file From de6da1d2920ae59178df29f2709fb265071b99bc Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:34:09 +0530 Subject: [PATCH 049/311] #OBS-I141: modified the url variable and access dataset_id from params --- api-service/src/metrics/prometheus/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 13d7f9d7..b87df8a7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -52,12 +52,12 @@ export const onGone = (req: any, res: Response) => { } const getMetricLabels = (req: any, res: Response) => { - const { id, entity, url, startTime } = req; + const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, "dataset_id") || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null const duration = getDuration(startTime); - const metricLabels = { entity, id, endpoint: url, dataset_id, status: statusCode, request_size, response_size } + const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } } From ea7b0f85ec0947c180b795d9f54ac67fd3751029 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:35:45 +0530 Subject: [PATCH 050/311] #OBS-I141: added helper function to get dataset_id for error cases --- api-service/src/helpers/ResponseHandler.ts | 4 ++-- api-service/src/metrics/prometheus/helpers.ts | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3bac60b7..e51eef27 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -1,7 +1,7 @@ import { NextFunction, Request, Response } from "express"; import httpStatus from "http-status"; import { IResponse, Result } from "../types/DatasetModels"; -import { onFailure, onSuccess } from "../metrics/prometheus/helpers"; +import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/helpers"; import moment from "moment"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; @@ -42,7 +42,7 @@ const ResponseHandler = { const resmsgid = _.get(res, "resmsgid") const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); - entity && onFailure(req, res) + entity && onObsrvFailure(req,res,error) }, setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index b87df8a7..2f502d99 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -2,6 +2,7 @@ import { NextFunction, Response } from "express"; import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; +import { ObsrvError } from "../../types/ObsrvError"; export const onRequest = ({ entity = Entity.Management }: any) => (req: any, res: Response, next: NextFunction) => { const startTime = Date.now(); @@ -51,6 +52,19 @@ export const onGone = (req: any, res: Response) => { incrementFailedApiCalls({ labels }); } +export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + metricLabels.dataset_id = error.datasetId + const { statusCode = 404 } = res + const labels = { ...metricLabels, status: statusCode } + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + const getMetricLabels = (req: any, res: Response) => { const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res From bc69d4b805f672cbddb99af9beb6820458837075 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:44:08 +0530 Subject: [PATCH 051/311] #OBS-I141: added telemetry for v2 api's --- api-service/src/app.ts | 3 +++ api-service/src/routes/Router.ts | 26 ++++++++++--------- api-service/src/services/telemetry.ts | 4 +-- api-service/src/telemetry/telemetryActions.ts | 5 +++- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index ec2a10cd..962331e9 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -8,13 +8,16 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; +import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) +app.use(interceptAuditEvents()); app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 3f2d623a..444a318d 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,16 +27,18 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; +import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import telemetryActions from "../telemetry/telemetryActions"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), dataIn); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), dataExhaust); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); @@ -45,14 +47,14 @@ router.patch("/template/update/:templateId", setDataToRequestObject("api.query.t router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); +router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), ConnectorsRead); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index ede157f7..e6139d94 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -4,7 +4,7 @@ import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import { Kafka } from "kafkajs"; -const env = _.get(appConfig, "env") +const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); @@ -29,7 +29,7 @@ const getDefaults = () => { sid: v4(), pdata: { id: `${env}.api.service`, - ver: "1.0.0" + ver: `${version}` } }, object: {}, diff --git a/api-service/src/telemetry/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts index bbee6faa..befe6100 100644 --- a/api-service/src/telemetry/telemetryActions.ts +++ b/api-service/src/telemetry/telemetryActions.ts @@ -17,5 +17,8 @@ export default { "sqlQuery": "dataset:query:sql", "ingestEvents": "dataset:events:ingest", "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" + "datasetExhaust": "dataset:exhaust:get", + "copyDataset": "dataset:copy", + "readConnectors": "connectors:read", + "listConnectors": "connectors:list", } \ No newline at end of file From c3e169737a3fb51d70edbb155c08523cf548dbb5 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:33:06 +0530 Subject: [PATCH 052/311] #OBS-I141: added a new metric to sum the response time --- api-service/src/metrics/prometheus/helpers.ts | 17 +++++++++++++---- api-service/src/metrics/prometheus/index.ts | 7 ++++--- api-service/src/metrics/prometheus/metrics.ts | 10 +++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 5e0442c1..13d7f9d7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -1,5 +1,5 @@ import { NextFunction, Response } from "express"; -import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; +import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; @@ -19,7 +19,10 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -28,7 +31,10 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -37,7 +43,10 @@ export const onGone = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 410 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } diff --git a/api-service/src/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts index da143f9f..3fdefe19 100644 --- a/api-service/src/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -1,6 +1,6 @@ import client from "prom-client"; -import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric } from "./metrics" -const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric]; +import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric } from "./metrics" +const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric]; import { NextFunction } from "express"; const register = new client.Registry(); @@ -17,6 +17,7 @@ const incrementApiCalls = ({ labels = {} }: Record) => totalApiCall const setQueryResponseTime = ({ labels = {}, duration }: Record) => queryResponseTimeMetric.labels(labels).set(duration); const incrementFailedApiCalls = ({ labels = {} }: Record) => failedApiCallsMetric.labels(labels).inc(); const incrementSuccessfulApiCalls = ({ labels = {} }: Record) => successApiCallsMetric.labels(labels).inc(); +const incrementResponseTime = ({ labels = {}, duration }: Record) => sumResponseTimeMetric.labels(labels).inc(duration); //register the metrics configureRegistry(register); @@ -31,5 +32,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls }; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls, incrementResponseTime}; diff --git a/api-service/src/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts index 2985dc0f..499bd15a 100644 --- a/api-service/src/metrics/prometheus/metrics.ts +++ b/api-service/src/metrics/prometheus/metrics.ts @@ -28,9 +28,17 @@ const successApiCallsMetric = new Prometheus.Counter({ labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] }) +// Create a new Prometheus Counter for sum of response time +const sumResponseTimeMetric = new Prometheus.Counter({ + name: "node_sum_response_time", + help: "The sum of response time for time series of same label", + labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] +}); + export { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, - successApiCallsMetric + successApiCallsMetric, + sumResponseTimeMetric } \ No newline at end of file From 195aaab864de2a0a99461c6f4f31f8ac13ff1c6a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:34:09 +0530 Subject: [PATCH 053/311] #OBS-I141: modified the url variable and access dataset_id from params --- api-service/src/metrics/prometheus/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 13d7f9d7..b87df8a7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -52,12 +52,12 @@ export const onGone = (req: any, res: Response) => { } const getMetricLabels = (req: any, res: Response) => { - const { id, entity, url, startTime } = req; + const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, "dataset_id") || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null const duration = getDuration(startTime); - const metricLabels = { entity, id, endpoint: url, dataset_id, status: statusCode, request_size, response_size } + const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } } From 7628bbd3b39fc29bc0144707d6acd6a7f4ee2f88 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:35:45 +0530 Subject: [PATCH 054/311] #OBS-I141: added helper function to get dataset_id for error cases --- api-service/src/helpers/ResponseHandler.ts | 4 ++-- api-service/src/metrics/prometheus/helpers.ts | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3bac60b7..e51eef27 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -1,7 +1,7 @@ import { NextFunction, Request, Response } from "express"; import httpStatus from "http-status"; import { IResponse, Result } from "../types/DatasetModels"; -import { onFailure, onSuccess } from "../metrics/prometheus/helpers"; +import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/helpers"; import moment from "moment"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; @@ -42,7 +42,7 @@ const ResponseHandler = { const resmsgid = _.get(res, "resmsgid") const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); - entity && onFailure(req, res) + entity && onObsrvFailure(req,res,error) }, setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index b87df8a7..2f502d99 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -2,6 +2,7 @@ import { NextFunction, Response } from "express"; import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; +import { ObsrvError } from "../../types/ObsrvError"; export const onRequest = ({ entity = Entity.Management }: any) => (req: any, res: Response, next: NextFunction) => { const startTime = Date.now(); @@ -51,6 +52,19 @@ export const onGone = (req: any, res: Response) => { incrementFailedApiCalls({ labels }); } +export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + metricLabels.dataset_id = error.datasetId + const { statusCode = 404 } = res + const labels = { ...metricLabels, status: statusCode } + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + const getMetricLabels = (req: any, res: Response) => { const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res From 2be8195a2a901e491c2a1d5b6770e9ca25efd2e4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:44:08 +0530 Subject: [PATCH 055/311] #OBS-I141: added telemetry for v2 api's --- api-service/src/app.ts | 3 +++ api-service/src/routes/Router.ts | 26 ++++++++++--------- api-service/src/services/telemetry.ts | 4 +-- api-service/src/telemetry/telemetryActions.ts | 5 +++- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index ec2a10cd..962331e9 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -8,13 +8,16 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; +import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) +app.use(interceptAuditEvents()); app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 3f2d623a..444a318d 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,16 +27,18 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; +import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import telemetryActions from "../telemetry/telemetryActions"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), dataIn); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), dataExhaust); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); @@ -45,14 +47,14 @@ router.patch("/template/update/:templateId", setDataToRequestObject("api.query.t router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); +router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), ConnectorsRead); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index ede157f7..e6139d94 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -4,7 +4,7 @@ import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import { Kafka } from "kafkajs"; -const env = _.get(appConfig, "env") +const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); @@ -29,7 +29,7 @@ const getDefaults = () => { sid: v4(), pdata: { id: `${env}.api.service`, - ver: "1.0.0" + ver: `${version}` } }, object: {}, diff --git a/api-service/src/telemetry/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts index bbee6faa..befe6100 100644 --- a/api-service/src/telemetry/telemetryActions.ts +++ b/api-service/src/telemetry/telemetryActions.ts @@ -17,5 +17,8 @@ export default { "sqlQuery": "dataset:query:sql", "ingestEvents": "dataset:events:ingest", "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" + "datasetExhaust": "dataset:exhaust:get", + "copyDataset": "dataset:copy", + "readConnectors": "connectors:read", + "listConnectors": "connectors:list", } \ No newline at end of file From ec87edc5e96bae83f6f518bc6db7ed7a26a508a7 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 16:32:57 +0530 Subject: [PATCH 056/311] #OBS-I141: added telemetry for v2 api's --- api-service/src/routes/Router.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 444a318d..1faaafb0 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -48,8 +48,8 @@ router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); -router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); From 0b051375d32f3263f19eb99ceb02e195eb2c41d8 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 9 Aug 2024 17:08:37 +0530 Subject: [PATCH 057/311] #OBS-I143: feat: dataset publish changes to deploy flink connectors --- .../{flink => flink-connector}/Chart.lock | 0 .../{flink => flink-connector}/Chart.yaml | 0 .../charts/.helmignore | 0 .../charts/common/Chart.yaml | 0 .../charts/common/templates/_affinities.tpl | 0 .../charts/common/templates/_capabilities.tpl | 0 .../charts/common/templates/_configs.tpl | 0 .../charts/common/templates/_errors.tpl | 0 .../charts/common/templates/_images.tpl | 0 .../charts/common/templates/_ingress.tpl | 0 .../charts/common/templates/_labels.tpl | 0 .../charts/common/templates/_names.tpl | 0 .../charts/common/templates/_secrets.tpl | 0 .../charts/common/templates/_storage.tpl | 0 .../charts/common/templates/_tplvalues.tpl | 0 .../charts/common/templates/_utils.tpl | 0 .../charts/common/templates/_variables.tpl | 0 .../charts/common/templates/_warnings.tpl | 0 .../templates/validations/_cassandra.tpl | 0 .../common/templates/validations/_mariadb.tpl | 0 .../common/templates/validations/_mongodb.tpl | 0 .../common/templates/validations/_mysql.tpl | 0 .../templates/validations/_postgresql.tpl | 0 .../common/templates/validations/_redis.tpl | 0 .../templates/validations/_validations.tpl | 0 .../charts/common/values.yaml | 0 .../templates/NOTES.txt | 0 .../templates/_base_serviceAccount.tpl | 0 .../templates/_helpers.tpl | 0 .../templates/_image_flink.tpl | 7 +-- .../templates/_namespace.tpl | 0 .../templates/configmap.yaml | 0 .../templates/deployment.yaml | 44 +++++----------- .../templates/hpa.yaml | 0 .../templates/ingress.yaml | 0 .../templates/service.yaml | 0 .../templates/serviceaccount.yaml | 0 .../templates/servicemonitor.yaml | 0 .../{flink => flink-connector}/values.yaml | 10 +--- .../src/command/connector_command.py | 52 ++++++++++++++++++- command-service/src/config/service_config.yml | 2 +- 41 files changed, 69 insertions(+), 46 deletions(-) rename command-service/helm-charts/{flink => flink-connector}/Chart.lock (100%) rename command-service/helm-charts/{flink => flink-connector}/Chart.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/.helmignore (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/Chart.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_affinities.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_capabilities.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_configs.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_errors.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_images.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_ingress.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_labels.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_names.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_secrets.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_storage.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_tplvalues.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_utils.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_variables.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_warnings.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_cassandra.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_mariadb.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_mongodb.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_mysql.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_postgresql.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_redis.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_validations.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/values.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/NOTES.txt (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/_base_serviceAccount.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/_helpers.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/_image_flink.tpl (57%) rename command-service/helm-charts/{flink => flink-connector}/templates/_namespace.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/configmap.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/deployment.yaml (83%) rename command-service/helm-charts/{flink => flink-connector}/templates/hpa.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/ingress.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/service.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/serviceaccount.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/servicemonitor.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/values.yaml (97%) diff --git a/command-service/helm-charts/flink/Chart.lock b/command-service/helm-charts/flink-connector/Chart.lock similarity index 100% rename from command-service/helm-charts/flink/Chart.lock rename to command-service/helm-charts/flink-connector/Chart.lock diff --git a/command-service/helm-charts/flink/Chart.yaml b/command-service/helm-charts/flink-connector/Chart.yaml similarity index 100% rename from command-service/helm-charts/flink/Chart.yaml rename to command-service/helm-charts/flink-connector/Chart.yaml diff --git a/command-service/helm-charts/flink/charts/.helmignore b/command-service/helm-charts/flink-connector/charts/.helmignore similarity index 100% rename from command-service/helm-charts/flink/charts/.helmignore rename to command-service/helm-charts/flink-connector/charts/.helmignore diff --git a/command-service/helm-charts/flink/charts/common/Chart.yaml b/command-service/helm-charts/flink-connector/charts/common/Chart.yaml similarity index 100% rename from command-service/helm-charts/flink/charts/common/Chart.yaml rename to command-service/helm-charts/flink-connector/charts/common/Chart.yaml diff --git a/command-service/helm-charts/flink/charts/common/templates/_affinities.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_affinities.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_configs.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_configs.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_errors.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_errors.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_images.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_images.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_ingress.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_ingress.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_labels.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_labels.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_names.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_names.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_secrets.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_secrets.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_storage.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_storage.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_utils.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_utils.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_variables.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_variables.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_warnings.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_warnings.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl diff --git a/command-service/helm-charts/flink/charts/common/values.yaml b/command-service/helm-charts/flink-connector/charts/common/values.yaml similarity index 100% rename from command-service/helm-charts/flink/charts/common/values.yaml rename to command-service/helm-charts/flink-connector/charts/common/values.yaml diff --git a/command-service/helm-charts/flink/templates/NOTES.txt b/command-service/helm-charts/flink-connector/templates/NOTES.txt similarity index 100% rename from command-service/helm-charts/flink/templates/NOTES.txt rename to command-service/helm-charts/flink-connector/templates/NOTES.txt diff --git a/command-service/helm-charts/flink/templates/_base_serviceAccount.tpl b/command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl similarity index 100% rename from command-service/helm-charts/flink/templates/_base_serviceAccount.tpl rename to command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl diff --git a/command-service/helm-charts/flink/templates/_helpers.tpl b/command-service/helm-charts/flink-connector/templates/_helpers.tpl similarity index 100% rename from command-service/helm-charts/flink/templates/_helpers.tpl rename to command-service/helm-charts/flink-connector/templates/_helpers.tpl diff --git a/command-service/helm-charts/flink/templates/_image_flink.tpl b/command-service/helm-charts/flink-connector/templates/_image_flink.tpl similarity index 57% rename from command-service/helm-charts/flink/templates/_image_flink.tpl rename to command-service/helm-charts/flink-connector/templates/_image_flink.tpl index d0d7c2da..5c542f64 100644 --- a/command-service/helm-charts/flink/templates/_image_flink.tpl +++ b/command-service/helm-charts/flink-connector/templates/_image_flink.tpl @@ -3,12 +3,13 @@ {{- $context := .context }} {{- $scope := .scope }} {{- with $scope }} -{{- $registry := default $context.Values.global.image.registry .registry }} -{{- $image := printf "%s/%s" $registry .repository}} +{{- $registry := default $context.Values.registry .registry }} +{{- $repository := default $context.Values.repository .repository }} +{{- $image := printf "%s/%s" $registry $repository}} {{- if .digest }} {{- printf "%s@%s" $image .digest }} {{- else }} -{{- $tag := default "latest" .tag }} +{{- $tag := default $context.Values.tag .tag }} {{- printf "%s:%s" $image $tag }} {{- end }} {{- end }} diff --git a/command-service/helm-charts/flink/templates/_namespace.tpl b/command-service/helm-charts/flink-connector/templates/_namespace.tpl similarity index 100% rename from command-service/helm-charts/flink/templates/_namespace.tpl rename to command-service/helm-charts/flink-connector/templates/_namespace.tpl diff --git a/command-service/helm-charts/flink/templates/configmap.yaml b/command-service/helm-charts/flink-connector/templates/configmap.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/configmap.yaml rename to command-service/helm-charts/flink-connector/templates/configmap.yaml diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml similarity index 83% rename from command-service/helm-charts/flink/templates/deployment.yaml rename to command-service/helm-charts/flink-connector/templates/deployment.yaml index 1cd613b3..d6d7708f 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -51,37 +51,17 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: {{ .Values.taskmanager.flink_work_dir }} - # args: ["taskmanager"] - # env: - # - name: FLINK_PROPERTIES - # value: |+ - # jobmanager.rpc.address: {{ $jobName }}-jobmanager - # taskmanager.rpc.port=6122 - # taskmanager.numberOfTaskSlots: 2 - # metrics.reporters: prom - # metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory - # metrics.reporter.prom.host: {{ $jobName }}-taskmanager - # metrics.reporter.prom.port: 9251 - command: ["/opt/flink/bin/taskmanager.sh"] - args: ["start-foreground", - {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}", - {{- end }} - {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} - "-Ds3.access-key={{ .Values.s3_access_key }}", - "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcs" }} - "-Dgoogle.cloud.auth.service.account.enable=true", - {{- end }} - "-Dweb.submit.enable=false", - "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", - "-Dmetrics.reporter.prom.host={{ $jobName }}-taskmanager", - "-Dmetrics.reporter.prom.port=9251-9260", - "-Djobmanager.rpc.address={{ $jobName }}-jobmanager", - "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] + args: ["taskmanager"] + env: + - name: FLINK_PROPERTIES + value: |+ + jobmanager.rpc.address: {{ $jobName }}-jobmanager + taskmanager.rpc.port=6122 + taskmanager.numberOfTaskSlots: 2 + metrics.reporters: prom + metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + metrics.reporter.prom.host: {{ $jobName }}-taskmanager + metrics.reporter.prom.port: 9251 ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc @@ -140,7 +120,7 @@ spec: app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} component: {{ printf "%s-%s" $jobName $component }} annotations: - checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/config: {{ .Files.Glob "cFonfigs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} spec: serviceAccountName: {{ include "base.serviceaccountname" . }} diff --git a/command-service/helm-charts/flink/templates/hpa.yaml b/command-service/helm-charts/flink-connector/templates/hpa.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/hpa.yaml rename to command-service/helm-charts/flink-connector/templates/hpa.yaml diff --git a/command-service/helm-charts/flink/templates/ingress.yaml b/command-service/helm-charts/flink-connector/templates/ingress.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/ingress.yaml rename to command-service/helm-charts/flink-connector/templates/ingress.yaml diff --git a/command-service/helm-charts/flink/templates/service.yaml b/command-service/helm-charts/flink-connector/templates/service.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/service.yaml rename to command-service/helm-charts/flink-connector/templates/service.yaml diff --git a/command-service/helm-charts/flink/templates/serviceaccount.yaml b/command-service/helm-charts/flink-connector/templates/serviceaccount.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/serviceaccount.yaml rename to command-service/helm-charts/flink-connector/templates/serviceaccount.yaml diff --git a/command-service/helm-charts/flink/templates/servicemonitor.yaml b/command-service/helm-charts/flink-connector/templates/servicemonitor.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/servicemonitor.yaml rename to command-service/helm-charts/flink-connector/templates/servicemonitor.yaml diff --git a/command-service/helm-charts/flink/values.yaml b/command-service/helm-charts/flink-connector/values.yaml similarity index 97% rename from command-service/helm-charts/flink/values.yaml rename to command-service/helm-charts/flink-connector/values.yaml index f0ea188c..84842bd9 100644 --- a/command-service/helm-charts/flink/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -321,15 +321,7 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom -flink_jobs: - kafka-connector: - enabled: true - registry: surabhi1510 - repository: flink-python-3.11 - tag: 1.0.0 - imagePullSecrets: [] - - job_classname: org.sunbird.obsrv.connector.KafkaConnector +flink_jobs: [] commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 2dbed28d..be82ae7b 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -86,12 +86,13 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None - for connector in active_connectors: print("Installing connector {0}".format(connector)) if connector.connector_runtime == "spark": result = self._perform_spark_install(connector) + elif connector.connector_runtime == "flink": + result = self._perform_flink_install(connector) else: print( f"Connector {connector.connector_id} is not supported for deployment" @@ -104,6 +105,55 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): # for release in masterdata_jar_config: # result = self._perform_install(release) return result + + def _perform_flink_install(self, connector_instance): + err = None + result = None + release_name = connector_instance.id + connector_source = json.loads(connector_instance.connector_source) + flink_jobs = { + "kafka-connector": { + "enabled": "true", + "job_classname": connector_source.get('main_class') + } + } + set_json_value = json.dumps(flink_jobs) + print("Kafka connector: ", set_json_value) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + self.connector_job_config["flink"]["namespace"], + "--create-namespace", + "--set-json", + f"flink_jobs={set_json_value}" + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + ) + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result def _perform_spark_install(self, connector_instance): err = None diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index a26d4a53..2c432ba0 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -172,7 +172,7 @@ connector_jobs: namespace: spark base_helm_chart: spark-connector-cron flink: - namespace: streaming-connectors + namespace: flink base_helm_chart: flink-connector connector_registry: From c09c6f3f70816be9095ce7b9f4dd6894af28790a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 12 Aug 2024 15:02:24 +0530 Subject: [PATCH 058/311] #OBS-I141: removed metric for sum of response time --- api-service/src/metrics/prometheus/helpers.ts | 22 +++++-------------- api-service/src/metrics/prometheus/index.ts | 7 +++--- api-service/src/metrics/prometheus/metrics.ts | 10 +-------- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 2f502d99..05051461 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -1,5 +1,5 @@ import { NextFunction, Response } from "express"; -import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; +import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; import { ObsrvError } from "../../types/ObsrvError"; @@ -20,10 +20,7 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -32,10 +29,7 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -44,10 +38,7 @@ export const onGone = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 410 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -57,10 +48,7 @@ export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { metricLabels.dataset_id = error.datasetId const { statusCode = 404 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } diff --git a/api-service/src/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts index 3fdefe19..173527ad 100644 --- a/api-service/src/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -1,6 +1,6 @@ import client from "prom-client"; -import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric } from "./metrics" -const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric]; +import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric } from "./metrics" +const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric]; import { NextFunction } from "express"; const register = new client.Registry(); @@ -17,7 +17,6 @@ const incrementApiCalls = ({ labels = {} }: Record) => totalApiCall const setQueryResponseTime = ({ labels = {}, duration }: Record) => queryResponseTimeMetric.labels(labels).set(duration); const incrementFailedApiCalls = ({ labels = {} }: Record) => failedApiCallsMetric.labels(labels).inc(); const incrementSuccessfulApiCalls = ({ labels = {} }: Record) => successApiCallsMetric.labels(labels).inc(); -const incrementResponseTime = ({ labels = {}, duration }: Record) => sumResponseTimeMetric.labels(labels).inc(duration); //register the metrics configureRegistry(register); @@ -32,5 +31,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls, incrementResponseTime}; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls}; diff --git a/api-service/src/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts index 499bd15a..2985dc0f 100644 --- a/api-service/src/metrics/prometheus/metrics.ts +++ b/api-service/src/metrics/prometheus/metrics.ts @@ -28,17 +28,9 @@ const successApiCallsMetric = new Prometheus.Counter({ labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] }) -// Create a new Prometheus Counter for sum of response time -const sumResponseTimeMetric = new Prometheus.Counter({ - name: "node_sum_response_time", - help: "The sum of response time for time series of same label", - labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] -}); - export { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, - successApiCallsMetric, - sumResponseTimeMetric + successApiCallsMetric } \ No newline at end of file From 59238bd2b8ee25a2783c3b7b86c1062da1ad54c4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 12 Aug 2024 15:05:13 +0530 Subject: [PATCH 059/311] #OBS-I141: removed usage of builtin kafka methods from telemetry file --- api-service/src/services/telemetry.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index e6139d94..d4eca61b 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -2,17 +2,13 @@ import { Request, Response, NextFunction } from "express" import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; -import { Kafka } from "kafkajs"; +import {send} from "../connections/kafkaConnection" const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); -const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const kafka = new Kafka({ clientId: telemetryTopic, brokers: brokerServers }); -const telemetryEventsProducer = kafka.producer(); -telemetryEventsProducer.connect().catch(err => console.error("Unable to connect to kafka", err.message)); const getDefaults = () => { return { @@ -54,7 +50,7 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { - telemetryEventsProducer.send({ topic: telemetryTopic, messages: [{ value: JSON.stringify(event) }] }).catch(console.log) + send({ messages: [{ value: JSON.stringify(event) }] }, telemetryTopic).catch(console.log); } const transformProps = (body: Record) => { From 21a97a43e8a0ed2b8441d37244f99247730ef4a6 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 12 Aug 2024 17:34:53 +0530 Subject: [PATCH 060/311] #OBS-I146: feat: Retire fix --- .../DatasetRetire.spec.ts | 92 ++----------------- .../DatasetStatusTransition/Fixtures.ts | 75 +++++++-------- 2 files changed, 40 insertions(+), 127 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index c5c32929..0be1fdb9 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -29,7 +29,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { it("Dataset status transition success: When the action is to Retire dataset", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE) }) chai.spy.on(DatasetTransformations, "update", () => { return Promise.resolve({}) @@ -44,7 +44,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { return Promise.resolve({}) }) chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) + return Promise.resolve([{ datasource_ref: "telemetry" }]) }) chai.spy.on(druidHttpService, "post", () => { return Promise.resolve({}) @@ -193,19 +193,19 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { it("Dataset status transition failure: When dataset is already retired", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Retired", type: "dataset" }) + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE, status: "Retired" }) }) chai .request(app) .post("/v2/datasets/status-transition") .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); + res.should.have.status(httpStatus.CONFLICT); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to Retire dataset as it is not in live state") + res.body.error.message.should.be.eq("Transition failed for dataset: dataset-all-fields7 status:Retired with status transition to Retire") res.body.error.code.should.be.eq("DATASET_RETIRE_FAILURE") done(); }) @@ -213,19 +213,13 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { it("Dataset status transition failure: When dataset to retire is used by other datasets", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", type: "master-dataset", status: "Live" }) + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE, type: "master" }) }) chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "telemetry" }] } }]) + return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "dataset-all-fields7" }] } }]) }) chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "telemetry" }] } }]) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) + return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "dataset-all-fields7" }] } }]) }) chai .request(app) @@ -237,78 +231,10 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to retire dataset as it is used by other datasets") + res.body.error.message.should.be.eq("Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset") res.body.error.code.should.be.eq("DATASET_IN_USE") done(); }); }); - it("Dataset status transition failure: When setting retire status to live records fail", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq("api.datasets.status-transition"); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); - - it("Dataset status transition failure: Failed to restart pipeline", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", type: "dataset", status: "Live", }) - }) - chai.spy.on(DatasetTransformations, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfig, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq("api.datasets.status-transition"); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index db14c924..9a807e7f 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -86,51 +86,38 @@ export const TestInputsForDatasetStatusTransition = { "entry_topic": "local.ingest" }, INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "", + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "tags": ["tag1"], + "status":"Draft", + "version_key": "1721887933020", + "api_version": "v2" + }, + SCHEMA_TO_RETIRE: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "tags": ["tag1"], + "status":"Live", + "version_key": "1721887933020", + "api_version": "v2" }, DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From 58643d41878fd04fb0adcd1b0fa998666f1a149e Mon Sep 17 00:00:00 2001 From: Aniket Sakinala Date: Tue, 13 Aug 2024 17:24:59 +0530 Subject: [PATCH 061/311] Issue #SBCOSS-12 fix: convert all SQL raw queries to prepared statements --- .../src/command/alert_manager_command.py | 10 +- .../src/command/connector_command.py | 31 +- .../src/command/connector_registry.py | 105 +++-- .../src/command/dataset_command.py | 17 +- command-service/src/command/db_command.py | 381 ++++++++++++------ command-service/src/command/druid_command.py | 3 +- command-service/src/service/db_service.py | 16 +- 7 files changed, 375 insertions(+), 188 deletions(-) diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 880fde1c..1c989a5c 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -81,13 +81,15 @@ def execute(self, command_payload: CommandPayload, action: Action): return ActionResponse(status="OK", status_code=200) def get_dataset(self, dataset_id: str) -> str: - query = f"SELECT * FROM datasets WHERE dataset_id='{dataset_id}'" - result = self.db_service.execute_select_one(sql=query) + query = f"SELECT * FROM datasets WHERE dataset_id= %s" + params = (dataset_id,) + result = self.db_service.execute_select_one(sql=query, params=params) return result def get_dataset_source_config(self, dataset_id: str) -> str: - query = f"SELECT * FROM dataset_source_config WHERE dataset_id='{dataset_id}'" - result = self.db_service.execute_select_all(sql=query) + query = f"SELECT * FROM dataset_source_config WHERE dataset_id= %s" + params = (dataset_id,) + result = self.db_service.execute_select_all(sql=query, params=params) return result def get_modified_metric( diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index be82ae7b..9b88a522 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -210,14 +210,14 @@ def _perform_spark_install(self, connector_instance): def _get_connector_details(self, dataset_id): active_connectors = [] - records = self.db_service.execute_select_all( - f""" - SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology - FROM connector_instances ci - JOIN connector_registry cr on ci.connector_id = cr.id - WHERE ci.status='{DatasetStatusType.Live.name}' and ci.dataset_id = '{dataset_id}' - """ - ) + query = f""" + SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology + FROM connector_instances ci + JOIN connector_registry cr on ci.connector_id = cr.id + WHERE ci.status= %s and ci.dataset_id = %s + """ + params = (DatasetStatusType.Live.name, dataset_id,) + records = self.db_service.execute_select_all(sql=query, params=params) for record in records: active_connectors.append(from_dict( @@ -228,14 +228,13 @@ def _get_connector_details(self, dataset_id): def _get_masterdata_details(self, dataset_id): is_masterdata = False - rows = self.db_service.execute_select_all( - f""" - SELECT * - FROM datasets - WHERE status='{DatasetStatusType.Live.name}' AND dataset_id = '{dataset_id}' AND type = 'master' - """ - ) - + query = f""" + SELECT * + FROM datasets + WHERE status= %s AND dataset_id = %s AND type = 'master' + """ + params = (DatasetStatusType.Live.name, dataset_id,) + rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: is_masterdata = True diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 36cb0b57..ae185174 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -205,8 +205,8 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S") ) - query = self.build_insert_query(registry_meta) - success = self.execute_query(query) + query, params = self.build_insert_query(registry_meta) + success = self.execute_query(query, params) if not success: return RegistryResponse( status="failure", @@ -254,8 +254,8 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S") ) - query = self.build_insert_query(registry_meta) - success = self.execute_query(query) + query, params = self.build_insert_query(registry_meta) + success = self.execute_query(query, params) if not success: return RegistryResponse( status="failure", @@ -269,9 +269,9 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_200_OK, ) - def execute_query(self, query) -> bool: + def execute_query(self, query, params) -> bool: try: - result = self.db_service.execute_upsert(query) + result = self.db_service.execute_upsert(sql=query, params=params) return result > 0 # Assuming the result is the number of affected rows except Exception as e: print( @@ -307,28 +307,79 @@ def download_file(self, url, destination) -> bool: def build_insert_query(self, registry_meta: ConnectorRegsitryv2): ui_spec_json = json.dumps(registry_meta.ui_spec) - return f""" INSERT INTO connector_registry (id, connector_id, name, type, category, version, description, technology, runtime, licence, owner, iconurl, status, source_url, source, ui_spec, created_by, updated_by, created_date, updated_date, live_date) VALUES - ( '{registry_meta.id}-{registry_meta.version}', '{registry_meta.id}', '{registry_meta.name}', '{registry_meta.type}', '{registry_meta.category}', '{registry_meta.version}', '{registry_meta.description}', '{registry_meta.technology}', '{registry_meta.runtime}', '{registry_meta.licence}', '{registry_meta.owner}', '{registry_meta.iconurl}', '{registry_meta.status}', '{registry_meta.source_url}', '{registry_meta.source}', '{ui_spec_json}', 'SYSTEM', 'SYSTEM', '{datetime.now()}', '{datetime.now()}', '{datetime.now()}' ) - ON CONFLICT (connector_id, version) DO UPDATE - SET id = '{registry_meta.id}-{registry_meta.version}', - name = '{registry_meta.name}', - type = '{registry_meta.type}', - category = '{registry_meta.category}', - version = '{registry_meta.version}', - description = '{registry_meta.description}', - technology = '{registry_meta.technology}', - runtime = '{registry_meta.runtime}', - licence = '{registry_meta.licence}', - owner = '{registry_meta.owner}', - iconurl = '{registry_meta.iconurl}', - status = '{registry_meta.status}', - source_url = '{registry_meta.source_url}', - source = '{registry_meta.source}', - ui_spec = '{ui_spec_json}'::jsonb, - updated_date = '{datetime.now()}' - ;; + query =f""" + INSERT INTO connector_registry ( + id, connector_id, name, type, category, version, description, + technology, runtime, licence, owner, iconurl, status, source_url, + source, ui_spec, created_by, updated_by, created_date, updated_date, live_date + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s + ) ON CONFLICT ( + connector_id, version + ) DO UPDATE SET + id = %s, + name = %s, + type = %s, + category = %s, + version = %s, + description = %s, + technology = %s, + runtime = %s, + licence = %s, + owner = %s, + iconurl = %s, + status = %s, + source_url = %s, + source = %s, + ui_spec = %s::jsonb, + updated_date = %s + ;; """ - + params = ( + registry_meta.id + "-" + registry_meta.version, + registry_meta.id, + registry_meta.name, + registry_meta.type, + registry_meta.category, + registry_meta.version, + registry_meta.description, + + registry_meta.technology, + registry_meta.runtime, + registry_meta.licence, + registry_meta.owner, + registry_meta.iconurl, + registry_meta.status, + registry_meta.source_url, + + registry_meta.source, + ui_spec_json, + 'SYSTEM', + 'SYSTEM', + datetime.now(), + datetime.now(), + datetime.now(), + + registry_meta.id, + registry_meta.name, + registry_meta.type, + registry_meta.category, + registry_meta.version, + registry_meta.description, + registry_meta.technology, + registry_meta.runtime, + registry_meta.licence, + registry_meta.owner, + registry_meta.iconurl, + registry_meta.status, + registry_meta.source_url, + registry_meta.source, + ui_spec_json, + datetime.now(), + ) + return query, params class ExtractionUtil: def extract_gz(tar_path, extract_path): diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index 165efd38..db7afa7d 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -30,9 +30,9 @@ def __init__( def _get_draft_dataset_record(self, dataset_id): query = f""" - SELECT "type", MAX(version) AS max_version FROM datasets_draft WHERE dataset_id = '{dataset_id}' GROUP BY 1 + SELECT "type", MAX(version) AS max_version FROM datasets_draft WHERE dataset_id = %s GROUP BY 1 """ - dataset_record = self.db_service.execute_select_one(query) + dataset_record = self.db_service.execute_select_one(sql=query, params=(dataset_id,)) if dataset_record is not None: return dataset_record return None @@ -40,19 +40,22 @@ def _get_draft_dataset_record(self, dataset_id): def _get_draft_dataset(self, dataset_id): query = f""" SELECT * FROM datasets_draft - WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}') AND version = (SELECT MAX(version) - FROM datasets_draft WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}')) + WHERE dataset_id = %s AND (status = %s OR status = %s ) AND version = (SELECT MAX(version) + FROM datasets_draft WHERE dataset_id = %s AND (status = %s OR status = %s )) """ - dataset_record = self.db_service.execute_select_one(query) + params = (dataset_id, DatasetStatusType.Publish.name, DatasetStatusType.ReadyToPublish.name, + dataset_id, DatasetStatusType.Publish.name, DatasetStatusType.ReadyToPublish.name,) + dataset_record = self.db_service.execute_select_one(sql=query, params=params) if dataset_record is not None: return dataset_record return None def _check_for_live_record(self, dataset_id): query = f""" - SELECT * FROM datasets WHERE dataset_id = '{dataset_id}' AND status = '{DatasetStatusType.Live.name}' + SELECT * FROM datasets WHERE dataset_id = %s AND status = %s """ - result = self.db_service.execute_select_one(query) + params = (dataset_id, DatasetStatusType.Live.name, ) + result = self.db_service.execute_select_one(sql=query, params=params) live_dataset = dict() if result is not None: live_dataset = from_dict(data_class=DatasetsLive, data=result) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 6d2c7d46..831300be 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -63,56 +63,99 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d draft_dataset = from_dict(data_class = DatasetsDraft, data = draft_dataset_record) draft_dataset_id = draft_dataset.id current_timestamp = dt.now() + params = ( + dataset_id, + dataset_id, + draft_dataset.type, + draft_dataset.name, + json.dumps(draft_dataset.extraction_config).replace("'", "''"), + json.dumps(draft_dataset.validation_config).replace("'", "''"), + json.dumps(draft_dataset.dedup_config).replace("'", "''"), + json.dumps(draft_dataset.denorm_config).replace("'", "''"), + json.dumps(draft_dataset.data_schema).replace("'", "''"), + json.dumps(draft_dataset.router_config).replace("'", "''"), + json.dumps(draft_dataset.dataset_config).replace("'", "''"), + DatasetStatusType.Live.name, + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + draft_dataset.api_version, + draft_dataset.version, + json.dumps(draft_dataset.sample_data).replace("'", "''"), + draft_dataset.entry_topic, + draft_dataset.created_by, + draft_dataset.updated_by, + current_timestamp, + current_timestamp, + current_timestamp, + + draft_dataset.name, + json.dumps(draft_dataset.extraction_config).replace("'", "''"), + json.dumps(draft_dataset.validation_config).replace("'", "''"), + json.dumps(draft_dataset.dedup_config).replace("'", "''"), + json.dumps(draft_dataset.denorm_config).replace("'", "''"), + json.dumps(draft_dataset.data_schema).replace("'", "''"), + json.dumps(draft_dataset.router_config).replace("'", "''"), + json.dumps(draft_dataset.dataset_config).replace("'", "''"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + data_version if live_dataset is not None else 1, + draft_dataset.api_version, + draft_dataset.version, + json.dumps(draft_dataset.sample_data).replace("'", "''"), + draft_dataset.entry_topic, + draft_dataset.updated_by, + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO datasets(id, dataset_id, "type", name, extraction_config, validation_config, dedup_config, denorm_config, data_schema, router_config, dataset_config, status, tags, data_version, api_version, version, sample_data, entry_topic, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{dataset_id}', - '{dataset_id}', - '{draft_dataset.type}', - '{draft_dataset.name}', - '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', - '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', - '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', - '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', - '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', - '{json.dumps(draft_dataset.router_config).replace("'", "''")}', - '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, 1, - '{draft_dataset.api_version}', - {draft_dataset.version}, - '{json.dumps(draft_dataset.sample_data).replace("'", "''")}', - '{draft_dataset.entry_topic}', - '{draft_dataset.created_by}', - '{draft_dataset.updated_by}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %d, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET name = '{draft_dataset.name}', - extraction_config = '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', - validation_config = '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', - dedup_config = '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', - denorm_config = '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', - data_schema = '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', - router_config = '{json.dumps(draft_dataset.router_config).replace("'", "''")}', - dataset_config = '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', - tags = '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', - data_version = {data_version if live_dataset is not None else 1}, - api_version = '{draft_dataset.api_version}', - version = {draft_dataset.version}, - sample_data = '{json.dumps(draft_dataset.sample_data).replace("'", "''")}', - entry_topic = '{draft_dataset.entry_topic}', - updated_by = '{draft_dataset.updated_by}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - status = '{DatasetStatusType.Live.name}'; + SET name = %s, + extraction_config = %s, + validation_config = %s, + dedup_config = %s, + denorm_config = %s, + data_schema = %s, + router_config = %s, + dataset_config = %s, + tags = %s, + data_version = %d, + api_version = %s, + version = %d, + sample_data = %s, + entry_topic = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; """ - self.db_service.execute_upsert(insert_query) + self.db_service.execute_upsert(insert_query, params) print(f"Dataset {dataset_id} record inserted successfully...") return draft_dataset_id @@ -120,51 +163,84 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): result = {} draft_datasource_record = self.db_service.execute_select_all( - f"SELECT * FROM datasources_draft WHERE dataset_id = '{draft_dataset_id}'" + sql=f"SELECT * FROM datasources_draft WHERE dataset_id = %s", + params=(draft_dataset_id,) ) if draft_datasource_record is None: return result for record in draft_datasource_record: draft_datasource = from_dict(data_class=DatasourcesDraft, data=record) current_timestamp = dt.now() + params = ( + draft_datasource.id, + draft_datasource.datasource, + dataset_id, + draft_datasource.datasource_ref, + json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + draft_datasource.type, + json.dumps(draft_datasource.retention_period).replace("'", "''"), + json.dumps(draft_datasource.archival_policy).replace("'", "''"), + json.dumps(draft_datasource.purge_policy).replace("'", "''"), + json.dumps(draft_datasource.backup_config).replace("'", "''"), + DatasetStatusType.Live.name, + draft_datasource.created_by, + draft_datasource.updated_by, + current_timestamp, + current_timestamp, + current_timestamp, + json.dumps(draft_datasource.metadata).replace("'", "''"), + + draft_datasource.datasource, + json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + draft_datasource.type, + json.dumps(draft_datasource.retention_period).replace("'", "''"), + json.dumps(draft_datasource.archival_policy).replace("'", "''"), + json.dumps(draft_datasource.purge_policy).replace("'", "''"), + json.dumps(draft_datasource.backup_config).replace("'", "''"), + draft_datasource.updated_by, + current_timestamp, + current_timestamp, + json.dumps(draft_datasource.metadata).replace("'", "''"), + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, updated_date, published_date, metadata) VALUES ( - '{draft_datasource.id}', - '{draft_datasource.datasource}', - '{dataset_id}', - '{draft_datasource.datasource_ref}', - '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', - '{draft_datasource.type}', - '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', - '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', - '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', - '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{draft_datasource.created_by}', - '{draft_datasource.updated_by}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}', - '{json.dumps(draft_datasource.metadata).replace("'", "''")}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET datasource_ref = '{draft_datasource.datasource}', - ingestion_spec = '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', - type = '{draft_datasource.type}', - retention_period = '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', - archival_policy = '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', - purge_policy = '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', - backup_config = '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', - updated_by = '{draft_datasource.updated_by}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - metadata = '{json.dumps(draft_datasource.metadata).replace("'", "''")}', - status = '{DatasetStatusType.Live.name}'; + SET datasource_ref = %s, + ingestion_spec = %s, + type = %s, + retention_period = %s, + archival_policy = %s, + purge_policy = %s, + backup_config = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + metadata = %s, + status = %s; """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print( f"Datasource {draft_datasource.id} record inserted successfully..." ) @@ -183,63 +259,105 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): ) current_timestamp = dt.now() if connector_config.version == 'v2': + params = ( + connector_config.id, + dataset_id, + connector_config.connector_id, + json.dumps(connector_config.connector_config).replace("'", "''"), + json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.data_format, + DatasetStatusType.Live.name, + json.dumps(emptyJson), + json.dumps(emptyJson), + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + + json.dumps(connector_config.connector_config).replace("'", "''"), + json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.data_format, + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO connector_instances(id, dataset_id, connector_id, connector_config, operations_config, data_format, status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{connector_config.id}', - '{dataset_id}', - '{connector_config.connector_id}', - '{json.dumps(connector_config.connector_config).replace("'", "''")}', - '{json.dumps(connector_config.operations_config).replace("'", "''")}', - '{connector_config.data_format}', - '{DatasetStatusType.Live.name}', - '{json.dumps(emptyJson)}', - '{json.dumps(emptyJson)}', - '{draft_dataset_record.get('created_by')}', - '{draft_dataset_record.get('updated_by')}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', - operations_config = '{json.dumps(connector_config.operations_config).replace("'", "''")}', - data_format = '{connector_config.data_format}', - updated_by = '{draft_dataset_record.get('updated_by')}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - status = '{DatasetStatusType.Live.name}'; + SET connector_config = %s, + operations_config = %s, + data_format = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print( f"Connector[v2] Instance record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." ) else: + params = ( + connector_config.id, + dataset_id, + connector_config.connector_id, + json.dumps(connector_config.connector_config).replace("'", "''"), + DatasetStatusType.Live.name, + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + + json.dumps(connector_config.connector_config).replace("'", "''"), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, status, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{connector_config.id}', - '{dataset_id}', - '{connector_config.connector_id}', - '{json.dumps(connector_config.connector_config).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{draft_dataset_record.get('created_by')}', - '{draft_dataset_record.get('updated_by')}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', - updated_by = '{draft_dataset_record.get('updated_by')}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - status = '{DatasetStatusType.Live.name}'; + SET connector_config = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print( f"Connector[v1] record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." ) @@ -252,7 +370,7 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): result = {} current_timestamp = dt.now() # Delete existing transformations - self.db_service.execute_delete(f"""DELETE from dataset_transformations where dataset_id = '{dataset_id}'""") + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations where dataset_id = %s""", params=(dataset_id,)) print(f"Dataset Transformation for {dataset_id} are deleted successfully...") if draft_dataset_transformations_record is None: @@ -262,31 +380,44 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): transformation = from_dict( data_class=DatasetTransformationsDraft, data=record ) + params = ( + dataset_id + '_' + transformation.field_key, + dataset_id, + transformation.field_key, + json.dumps(transformation.transformation_function).replace("'", "''"), + DatasetStatusType.Live.name, + transformation.mode, + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + ) insert_query = f""" INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, status, mode, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{dataset_id + '_' + transformation.field_key}', - '{dataset_id}', - '{transformation.field_key}', - '{json.dumps(transformation.transformation_function).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{transformation.mode}', - '{draft_dataset_record.get('created_by')}', - '{draft_dataset_record.get('updated_by')}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") return result def _delete_draft_dataset(self, dataset_id, draft_dataset_id): - self.db_service.execute_delete(f"""DELETE from datasources_draft where dataset_id = '{draft_dataset_id}'""") + self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") - self.db_service.execute_delete(f"""DELETE from datasets_draft where id = '{draft_dataset_id}'""") + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index 1bd95ab2..901b18ad 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -27,7 +27,8 @@ def execute(self, command_payload: CommandPayload, action: Action): def _submit_ingestion_task(self, dataset_id): datasources_records = self.db_service.execute_select_all( - f"SELECT dso.*, dt.type as dataset_type FROM datasources dso, datasets dt WHERE dso.dataset_id = '{dataset_id}' AND dso.dataset_id = dt.id" + sql=f"SELECT dso.*, dt.type as dataset_type FROM datasources dso, datasets dt WHERE dso.dataset_id = %s AND dso.dataset_id = dt.id", + params=(dataset_id,) ) if datasources_records is not None: print( diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py index 864b4612..ded56b91 100644 --- a/command-service/src/service/db_service.py +++ b/command-service/src/service/db_service.py @@ -38,36 +38,36 @@ def connect(self): return db_connection # @reconnect - def execute_select_one(self, sql): + def execute_select_one(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) result = cursor.fetchone() db_connection.close() return result # @reconnect - def execute_select_all(self, sql): + def execute_select_all(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) result = cursor.fetchall() db_connection.close() return result # @reconnect - def execute_upsert(self, sql): + def execute_upsert(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) record_count = cursor.rowcount db_connection.close() # print(f"{record_count} inserted/updated successfully") return record_count # @reconnect - def execute_delete(self, sql): + def execute_delete(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) db_connection.close() From 2f2f7557d26137e73850085ff4ee85b1ed9b38e7 Mon Sep 17 00:00:00 2001 From: Aniket Sakinala Date: Mon, 19 Aug 2024 18:23:42 +0530 Subject: [PATCH 062/311] Issue #SBCOSS-12 fix: tags is an array, so requires empty json for null case; dataset draft deletion requires deletion of transformation and source config drafts --- command-service/src/command/db_command.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 831300be..4cb62e2c 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -76,7 +76,7 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d json.dumps(draft_dataset.router_config).replace("'", "''"), json.dumps(draft_dataset.dataset_config).replace("'", "''"), DatasetStatusType.Live.name, - json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}") if draft_dataset.tags is not None else json.dumps({}), draft_dataset.api_version, draft_dataset.version, json.dumps(draft_dataset.sample_data).replace("'", "''"), @@ -95,7 +95,7 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d json.dumps(draft_dataset.data_schema).replace("'", "''"), json.dumps(draft_dataset.router_config).replace("'", "''"), json.dumps(draft_dataset.dataset_config).replace("'", "''"), - json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}") if draft_dataset.tags is not None else json.dumps({}), data_version if live_dataset is not None else 1, draft_dataset.api_version, draft_dataset.version, @@ -126,7 +126,7 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d %s, 1, %s, - %d, + %s, %s, %s, %s, @@ -145,9 +145,9 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d router_config = %s, dataset_config = %s, tags = %s, - data_version = %d, + data_version = %s, api_version = %s, - version = %d, + version = %s, sample_data = %s, entry_topic = %s, updated_by = %s, @@ -419,5 +419,11 @@ def _delete_draft_dataset(self, dataset_id, draft_dataset_id): self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft source config/tables for {dataset_id} are deleted successfully...") + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file From 727dd2f92422bfa2a549e8b8159199453cab78e3 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 20 Aug 2024 11:30:25 +0530 Subject: [PATCH 063/311] #SBCOSS-23: feat: dataset publish changes for redeployment --- .../flink-connector/templates/deployment.yaml | 44 ++- .../helm-charts/flink-connector/values.yaml | 23 +- .../src/command/connector_command.py | 340 +++++++++++++----- 3 files changed, 280 insertions(+), 127 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index d6d7708f..946ca29d 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -69,9 +69,12 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - - name: flink-config-volume - mountPath: /data/flink/conf/flink-connector.conf - subPath: flink-connector.conf + # - name: flink-config-volume + # mountPath: /data/flink/conf/connectors-scala-config.conf + # subPath: connectors-scala-config.conf + # - name: flink-config-volume + # mountPath: /data/flink/conf/connectors-python-config.conf + # subPath: connectors-python-config.yaml # - name: flink-config-volume # mountPath: /opt/flink/conf/flink-conf.yaml # subPath: flink-conf.yaml @@ -79,12 +82,14 @@ spec: mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties volumes: - - name: flink-config-volume - configMap: - name: flink-connector-conf - items: - - key: connectors-scala-config.conf - path: flink-connector.conf + # - name: flink-config-volume + # configMap: + # name: flink-connector-conf + # items: + # - key: connectors-scala-config.conf + # path: connectors-scala-config.conf + # - key: connectors-python-config.yaml + # path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config @@ -151,17 +156,19 @@ spec: metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory metrics.reporter.prom.host: {{ $jobName }}-jobmanager metrics.reporter.prom.port: 9250 - volumeMounts: - name: flink-config-volume - mountPath: /data/flink/conf/flink-connector.conf - subPath: flink-connector.conf + mountPath: /data/flink/conf/connectors-scala-config.conf + subPath: connectors-scala-config.conf + - name: flink-config-volume + mountPath: /data/flink/conf/connectors-python-config.conf + subPath: connectors-python-config.conf - name: data mountPath: /flink/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - + - name: {{ $jobName }}-job-submit image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} @@ -199,8 +206,11 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - name: flink-config-volume - mountPath: /data/flink/conf/flink-connector.conf - subPath: flink-connector.conf + mountPath: /data/flink/conf/connectors-scala-config.conf + subPath: connectors-scala-config.conf + - name: flink-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml - name: data mountPath: /flink/connectors - name: flink-common-volume @@ -214,7 +224,9 @@ spec: name: flink-connector-conf items: - key: connectors-scala-config.conf - path: flink-connector.conf + path: connectors-scala-config.conf + - key: connectors-python-config.yaml + path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index 84842bd9..b1c48924 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -264,27 +264,6 @@ baseconfig: | port = "{{ .Values.global.cassandra.port }}" } -# !!! Don't override the resources here. It's just a template -# Proper way to override resouce is to -# flink_jobs: -# master-data-processor: -# resources: -# taskmanager: -# resources: -# requests: -# cpu: 100m -# memory: 100Mi -# limits: -# cpu: 1 -# memory: 1024Mi -# jobmanager: -# resources: -# requests: -# cpu: 100m -# memory: 100Mi -# limits: -# cpu: 1 -# memory: 1024Mi flink_resources: taskmanager: resources: @@ -303,7 +282,7 @@ flink_resources: cpu: 1 memory: 1024Mi -serviceMonitor: +serviceMonitor: jobmanager: enabled: true interval: 30s diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index be82ae7b..c24d6408 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -38,12 +38,16 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - # self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata) + namespaces = set() + for job_type, config in self.connector_job_config.items(): + namespaces.add(config["namespace"]) + for namespace in namespaces: + self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata, namespace) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result - def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): + def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata, namespace): managed_releases = [] connector_jar_config = self.config.find("connector_job") masterdata_jar_config = self.config.find("masterdata_job") @@ -54,7 +58,7 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): for release in masterdata_jar_config: managed_releases.append(release["release_name"]) - helm_ls_cmd = ["helm", "ls", "--namespace", self.connector_job_ns] + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -90,15 +94,15 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): print("Installing connector {0}".format(connector)) if connector.connector_runtime == "spark": - result = self._perform_spark_install(connector) + result = self._perform_spark_install(dataset_id, connector) elif connector.connector_runtime == "flink": - result = self._perform_flink_install(connector) + result = self._perform_flink_install(dataset_id, connector) else: print( f"Connector {connector.connector_id} is not supported for deployment" ) break - + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -106,113 +110,235 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): # result = self._perform_install(release) return result - def _perform_flink_install(self, connector_instance): + def _perform_flink_install(self, dataset_id, connector_instance): err = None result = None - release_name = connector_instance.id - connector_source = json.loads(connector_instance.connector_source) - flink_jobs = { - "kafka-connector": { - "enabled": "true", - "job_classname": connector_source.get('main_class') - } - } - set_json_value = json.dumps(flink_jobs) - print("Kafka connector: ", set_json_value) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", - "--namespace", - self.connector_job_config["flink"]["namespace"], - "--create-namespace", - "--set-json", - f"flink_jobs={set_json_value}" - ] + release_name = connector_instance.connector_id + runtime = connector_instance.connector_runtime + namespace = self.connector_job_config["flink"]["namespace"] - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] + + helm_ls_result = subprocess.run( + helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} deployment succeeded...") + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + print(jobs) + deployment_exists = any(release_name in line for line in jobs.splitlines()[1:]) + if deployment_exists: + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, release_name=release_name + ) + print("Restart command: ", restart_cmd) + # Run the helm command + helm_install_result = subprocess.run( + restart_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} re-deployment succeeded...") + else: + err = True + return ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_LIST_EXCEPTION", + ) + print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result + else: + connector_source = json.loads(connector_instance.connector_source) + flink_jobs = { + "kafka-connector": { + "enabled": "true", + "job_classname": connector_source.get('main_class') + } + } + set_json_value = json.dumps(flink_jobs) + print("Kafka connector: ", set_json_value) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set-json", + f"flink_jobs={set_json_value}" + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + ) + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result else: - err = True - result = ActionResponse( + print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") + return ActionResponse( status="ERROR", status_code=500, - error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + error_message="FLINK_HELM_LIST_EXCEPTION", ) - print( - f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" - ) - - if err is None: - result = ActionResponse(status="OK", status_code=200) - - return result - - def _perform_spark_install(self, connector_instance): + + def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None release_name = connector_instance.id - # print("Instance -->>", connector_instance) connector_source = json.loads(connector_instance.connector_source) - print(connector_source) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", - "--namespace", - self.connector_job_config["spark"]["namespace"], - "--create-namespace", - "--set", - "technology={}".format(connector_instance.technology), - "--set", - "instance_id={}".format(release_name), - "--set", - "connector_source={}".format(connector_source["source"]), - "--set", - "main_class={}".format(connector_source["main_class"]), - "--set", - "main_file={}".format(connector_source["main_program"]), - "--set", - "cronSchedule={}".format(connector_instance.operations_config["schedule"]) - ] - - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} deployment succeeded...") + schedule = connector_instance.operations_config["schedule"] + + schedule_configs = { + "Hourly": "0 * * * *", # Runs at the start of every hour + "Weekly": "0 0 * * 0", # Runs at midnight every Sunday + "Monthly": "0 0 1 * *", # Runs at midnight on the 1st day of every month + "Yearly": "0 0 1 1 *" # Runs at midnight on January 1st each year + } + + # Define namespace + namespace = self.connector_job_config["spark"]["namespace"] + + # Check if the job already exists + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] + helm_ls_result = subprocess.run(helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + job_exists = any(release_name in line for line in jobs.splitlines()[1:]) + + if job_exists: + if self._is_dataset_retired(dataset_id): + print("Dataset is retired. Uninstalling existing cron job.") + self._stop_connector_jobs(dataset_id, [], False, namespace) + else: + print("Updating existing job") + + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} update succeeded...") + result = ActionResponse(status="OK", status_code=200) + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", + ) + print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") + else: + print("Installing new job") + + helm_install_cmd = [ + "helm", + "install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} installation succeeded...") + result = ActionResponse(status="OK", status_code=200) + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", + ) + print(f"Error installing job {release_name}: {helm_install_result.stderr.decode()}") + else: - err = True + print(f"Error listing Spark jobs: {helm_ls_result.stderr.decode()}") result = ActionResponse( status="ERROR", status_code=500, - error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", - ) - print( - f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + error_message="SPARK_HELM_LIST_EXCEPTION", ) - - if err is None: - result = ActionResponse(status="OK", status_code=200) + + if result is None: + result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") return result def _get_connector_details(self, dataset_id): active_connectors = [] + connector_versions = {} records = self.db_service.execute_select_all( f""" - SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology + SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version FROM connector_instances ci JOIN connector_registry cr on ci.connector_id = cr.id WHERE ci.status='{DatasetStatusType.Live.name}' and ci.dataset_id = '{dataset_id}' @@ -223,6 +349,7 @@ def _get_connector_details(self, dataset_id): active_connectors.append(from_dict( data_class=ConnectorInstance, data=record )) + # connector_versions[connector_instance.id] = record['version'] return active_connectors @@ -240,3 +367,38 @@ def _get_masterdata_details(self, dataset_id): is_masterdata = True return is_masterdata + + def _is_dataset_retired(self, dataset_id): + is_retired = False + rows = self.db_service.execute_select_all( + f""" + SELECT status + FROM datasets + WHERE status='{DatasetStatusType.Retired.name}' AND dataset_id = '{dataset_id}' + """ + ) + for row in rows: + if row['status'] == DatasetStatusType.Retired.name: + print("Status: ",row['status']) + is_retired = True + break + + return is_retired + + def _get_live_instances(self, dataset_id, runtime, connector_instance): + active_connectors = [] + records = self.db_service.execute_select_all( + f""" + SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id + FROM connector_instances ci + JOIN connector_registry cr ON ci.connector_id = cr.id + JOIN datasets d ON ci.dataset_id = d.id + WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}'; + """ + ) + + for record in records: + active_connectors.append(record['connector_instance_id']) + + return active_connectors + From ce5ccff713b09e1e0549165ceb1f1977e8fc03b1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 20 Aug 2024 12:55:51 +0530 Subject: [PATCH 064/311] #OBS-I167 : read api changes while reading connectors according to v2 structure --- .../controllers/DatasetRead/DatasetRead.ts | 56 +++++++++++++++---- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 28909ff3..0752a013 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -39,36 +39,70 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ - ...connector, - connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) - })); + dataset.connectors_config = dataset?.connectors_config.map((connector: any) => { + const connector_config = _.get(connector, "connector_config") + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector.connector_config)) + } + }); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { - + const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); - if(draftDataset) { // Contains a draft + if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); - if(liveDataset) { + if (liveDataset) { const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } - + return null; } const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); - return dataset; + const api_version = _.get(dataset, "api_version") + let datasetConfigs: any = {} + const connectors_config: any[] = await datasetService.getConnectorsV1(datasetId, ["id", "dataset_id", "connector_config", "connector_type"]); + const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) + if (api_version !== "v2") { + datasetConfigs["connectors_config"] = _.map(connectors_config, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { + console.log(config) + const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); + return { + field_key: _.get(config, "field_key"), + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, "metadata._transformedFieldDataType") || _.get(config, "transformation_function.datatype") || "string", + category: datasetService.getTransformationCategory(section), + }, + mode: _.get(config, "mode") + } + }) + } + else { + datasetConfigs["connectors_config"] = connectors_config; + datasetConfigs["transformations_config"] = transformations_config; + } + return { ...dataset, ...datasetConfigs }; } export default datasetRead; \ No newline at end of file From 6da29abbfce53c8b9d63e9f91b0c0cf5ac63d743 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 13:16:39 +0530 Subject: [PATCH 065/311] #OBS-I173: fix: Ready to publish schema fix to expect connector configs as object and string --- .../DatasetStatusTransition/ReadyToPublishSchema.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 281c62ac..f378a5f7 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -376,7 +376,14 @@ "minLength": 1 }, "connector_config": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "operations_config": { "type": "object" From 03565483c9dbe1781e9130788dbc5c0353fb546c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 13:18:19 +0530 Subject: [PATCH 066/311] #OBS-I174: fix: Dataset read api fix to expect both v1 and v2 connectors --- api-service/src/services/DatasetService.ts | 103 +++++++++++---------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 25b120f2..c0ccaf1d 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -25,7 +25,7 @@ class DatasetService { } findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Dataset.findAll({where, attributes, order, raw: true}) + return Dataset.findAll({ where, attributes, order, raw: true }) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -39,9 +39,9 @@ class DatasetService { } checkDatasetExists = async (dataset_id: string): Promise => { - const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); if (draft === null) { - const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); return !(live === null) } else { return true; @@ -53,7 +53,7 @@ class DatasetService { } findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return DatasetDraft.findAll({where, attributes, order, raw: true}) + return DatasetDraft.findAll({ where, attributes, order, raw: true }) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -78,7 +78,7 @@ class DatasetService { updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; @@ -96,7 +96,7 @@ class DatasetService { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - + try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -150,9 +150,9 @@ class DatasetService { return draftDataset; } - getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string): string => { - switch(section) { + switch (section) { case "pii": return "pii"; case "additionalFields": @@ -163,15 +163,15 @@ class DatasetService { } createDraftDatasetFromLive = async (dataset: Model) => { - - let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); - const dataset_config:any = _.get(dataset, "dataset_config"); - const api_version:any = _.get(dataset, "api_version"); - if(api_version === "v1") { + + let draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { @@ -198,21 +198,30 @@ class DatasetService { draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { - const connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); - draftDataset["connectors_config"] = connectors + const v1connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); + const modifiedV1Connectors = _.map(v1connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); + draftDataset["connectors_config"] = _.concat(modifiedV1Connectors, v2connectors) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } const denormConfig = _.get(draftDataset, "denorm_config") if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) if (_.isEmpty(masterDatasets)) { throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } } const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { const { redis_db, denorm_out_field, denorm_key } = field let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) - if(!masterConfig){ + if (!masterConfig) { masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) } if (_.isEmpty(masterConfig)) { @@ -238,12 +247,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -255,18 +264,18 @@ class DatasetService { try { await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); - } catch(err:any) { + } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Datasource.findAll({where, attributes, order, raw: true}) + return Datasource.findAll({ where, attributes, order, raw: true }) } private deleteDruidSupervisors = async (dataset: Record) => { @@ -290,26 +299,26 @@ class DatasetService { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - if(indexingConfig.olap_store_enabled) { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { await this.createDruidDataSource(draftDataset, transaction); } - if(indexingConfig.lakehouse_enabled) { + if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if(liveDataset && liveDataset.api_version === "v2") { + if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } await transaction.commit() - } catch(err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } await executeCommand(draftDataset.id, "PUBLISH_DATASET"); - + } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -318,7 +327,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -327,25 +336,25 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") - const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } - private createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string): Record => { - const datasource = _.join([draftDataset.dataset_id,"events"], "_") + const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { - id: _.join([datasource,type], '_'), + id: _.join([datasource, type], '_'), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, @@ -356,15 +365,15 @@ class DatasetService { } export const getLiveDatasetConfigs = async (dataset_id: string) => { - + let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - if(!_.isEmpty(transformations)){ + if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } - if(!_.isEmpty(connectors)){ + if (!_.isEmpty(connectors)) { datasetRecord["connectors_config"] = connectors } return datasetRecord; From 67343d8af1d9eb6c8785836c2c3067021b5a9133 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 16:00:02 +0530 Subject: [PATCH 067/311] #OBS-I146: fix: Test case fix for read api --- .../controllers/DatasetRead/DatasetRead.ts | 12 +-- api-service/src/services/DatasetService.ts | 94 +++++++++---------- .../DatasetCreate/Fixtures.ts | 55 +++++++++++ .../DatasetRead/DatasetRead.spec.ts | 12 +-- .../DatasetUpdate/DatasetConnectors.spec.ts | 87 +++++++++++++++++ .../DatasetUpdate/Fixtures.ts | 16 ++++ 6 files changed, 217 insertions(+), 59 deletions(-) create mode 100644 api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index fc16b313..4265788d 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -48,21 +48,21 @@ const datasetRead = async (req: Request, res: Response) => { } const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { - + const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); - if(draftDataset) { // Contains a draft + if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); - if(liveDataset) { + if (liveDataset) { const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } - + return null; } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index f5298126..50b42668 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -25,7 +25,7 @@ class DatasetService { } findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Dataset.findAll({where, attributes, order, raw: true}) + return Dataset.findAll({ where, attributes, order, raw: true }) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -39,9 +39,9 @@ class DatasetService { } checkDatasetExists = async (dataset_id: string): Promise => { - const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); if (draft === null) { - const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); return !(live === null) } else { return true; @@ -53,7 +53,7 @@ class DatasetService { } findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return DatasetDraft.findAll({where, attributes, order, raw: true}) + return DatasetDraft.findAll({ where, attributes, order, raw: true }) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -78,7 +78,7 @@ class DatasetService { updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; @@ -96,7 +96,7 @@ class DatasetService { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - + try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -150,9 +150,9 @@ class DatasetService { return draftDataset; } - getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string): string => { - switch(section) { + switch (section) { case "pii": return "pii"; case "additionalFields": @@ -163,15 +163,15 @@ class DatasetService { } createDraftDatasetFromLive = async (dataset: Model) => { - - const draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); - const dataset_config:any = _.get(dataset, "dataset_config"); - const api_version:any = _.get(dataset, "api_version"); - if(api_version === "v1") { + + const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { @@ -205,14 +205,14 @@ class DatasetService { } const denormConfig = _.get(draftDataset, "denorm_config") if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) if (_.isEmpty(masterDatasets)) { throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } } const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { const { redis_db, denorm_out_field, denorm_key } = field let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) - if(!masterConfig){ + if (!masterConfig) { masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) } if (_.isEmpty(masterConfig)) { @@ -225,8 +225,8 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft - await DatasetDraft.create(draftDataset); - return await this.getDraftDataset(draftDataset.dataset_id); + const result = await DatasetDraft.create(draftDataset); + return _.get(result, "dataValues") } getNextRedisDBIndex = async () => { @@ -238,12 +238,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -255,18 +255,18 @@ class DatasetService { try { await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); - } catch(err:any) { + } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Datasource.findAll({where, attributes, order, raw: true}) + return Datasource.findAll({ where, attributes, order, raw: true }) } private deleteDruidSupervisors = async (dataset: Record) => { @@ -290,26 +290,26 @@ class DatasetService { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - if(indexingConfig.olap_store_enabled) { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { await this.createDruidDataSource(draftDataset, transaction); } - if(indexingConfig.lakehouse_enabled) { + if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if(liveDataset && liveDataset.api_version === "v2") { + if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } await transaction.commit() - } catch(err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } await executeCommand(draftDataset.id, "PUBLISH_DATASET"); - + } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -318,7 +318,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -327,25 +327,25 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") - const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } - private createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string): Record => { - const datasource = _.join([draftDataset.dataset_id,"events"], "_") + const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { - id: _.join([datasource,type], "_"), + id: _.join([datasource, type], "_"), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, @@ -356,15 +356,15 @@ class DatasetService { } export const getLiveDatasetConfigs = async (dataset_id: string) => { - + const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - if(!_.isEmpty(transformations)){ + if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } - if(!_.isEmpty(connectors)){ + if (!_.isEmpty(connectors)) { datasetRecord["connectors_config"] = connectors } return datasetRecord; diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts index 46aaebeb..01b87ac1 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -161,6 +161,54 @@ export const TestInputsForDatasetCreate = { } }, + VALID_DATASET_WITH_CONNECTORS: { + "id": "api.datasets.create", + "ver": "v1", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "dataset_id": "sb-ddd", + "type": "event", + "name": "sb-telemetry2", + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "eid": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ets": { + "type": "string" + }, + "required": [ + "eid" + ] + }, + "additionalProperties": true + }, + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "connectors_config":[{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, {"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5cai","connector_id":"debezium","connector_config":{"type":"debezium","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}], + "tags": [] + } + }, + VALID_MINIMAL_DATASET: { "id": "api.datasets.create", "ver": "v2", @@ -536,6 +584,13 @@ export const DATASET_CREATE_SUCCESS_FIXTURES = [ "status": "SUCCESS", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, + { + "title": "Dataset creation success: When connectors payload provided", + "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_CONNECTORS, + "httpStatus": httpStatus.OK, + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, { "title": "Dataset creation success: When multiple transformation payload provided with same field key", "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_MULTIPLE_TRANSFORMATIONS, diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 8b104ec9..ca0ec0d1 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -25,7 +25,7 @@ describe("DATASET READ API", () => { it("Dataset read success: When minimal fields requested", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "name": "sb-telemetry", "data_version": 1 }) + return Promise.resolve({ "name": "sb-telemetry", "version": 1 }) }) chai .request(app) @@ -38,7 +38,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: "sb-telemetry", data_version: 1 })) + result.should.be.eq(JSON.stringify({ name: "sb-telemetry", version: 1 })) done(); }); }); @@ -64,7 +64,7 @@ describe("DATASET READ API", () => { }); }); - it("Dataset read success: Fetch live dataset when status param is empty", (done) => { + it("Dataset read success: Fetch live dataset when mode param not provided", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) @@ -150,20 +150,20 @@ describe("DATASET READ API", () => { it("Dataset read failure: Updating dataset status to draft on mode=edit fails as live record not found", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "sb-telemetry", name: "sb-telemetry", status: "Live", data_schema: {} }) + return Promise.resolve() }) chai.spy.on(Dataset, "findOne", () => { return Promise.resolve() }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft&mode=edit") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.NOT_FOUND); res.body.should.be.a("object") res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to fetch live dataset") + res.body.error.message.should.be.eq("Dataset with the given dataset_id:sb-telemetry not found") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts new file mode 100644 index 00000000..34907197 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts @@ -0,0 +1,87 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import _ from "lodash"; +import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; +import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { sequelize } from "../../../connections/databaseConnection"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +describe("DATASET CONNECTORS UPDATE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Success: Dataset connectors successfully added", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", connectors_config:[] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_CONNECTORS_ADD) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + + it("Success: Dataset connectors successfully removed", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", connectors_config:[{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_CONNECTORS_REMOVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index eb2c683b..97248d9f 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -88,6 +88,22 @@ export const TestInputsForDatasetUpdate = { } }, + DATASET_UPDATE_CONNECTORS_ADD: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "connectors_config":[{"value":{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, "action": "upsert"}], + } + }, + + DATASET_UPDATE_CONNECTORS_REMOVE: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "connectors_config":[{"value":{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, "action": "upsert"}], + } + }, + DATASET_UPDATE_DEDUP_DUPLICATES_TRUE: { ...requestStructure, request: { From 680f67f88e9e170119de965946dd8bfc2fd89c36 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 18:23:03 +0530 Subject: [PATCH 068/311] #OBS-I146: fix: Test case fix for read api changes --- .../controllers/DatasetRead/DatasetRead.ts | 9 +-- .../DatasetRead/DatasetRead.spec.ts | 52 ++++++++++++- .../DatasetManagement/DatasetRead/Fixtures.ts | 78 +++++++++++++------ .../DatasetReadyToPublish.spec.ts | 13 +--- .../DatasetUpdate/DatasetConnectors.spec.ts | 13 ---- .../DatasetUpdate/DatasetDedup.spec.ts | 13 ---- .../DatasetUpdate/DatasetDenorm.spec.ts | 19 ----- .../DatasetUpdate/DatasetExtraction.spec.ts | 14 +--- .../DatasetUpdate/DatasetTags.spec.ts | 14 +--- .../DatasetTransformation.spec.ts | 21 +---- .../DatasetUpdate/DatasetUpdate.spec.ts | 14 +--- .../DatasetUpdate/DatasetValidation.spec.ts | 14 +--- 12 files changed, 119 insertions(+), 155 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 4265788d..2822e87b 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -11,16 +11,13 @@ export const apiId = "api.datasets.read"; export const errorCode = "DATASET_READ_FAILURE" // TODO: Move this to a config -const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] +export const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] const validateRequest = (req: Request) => { const { dataset_id } = req.params; - const fields = req.query.fields; - if (fields && typeof fields !== "string") { - throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); - } - const fieldValues = fields ? _.split(fields, ",") : []; + const { fields } = req.query; + const fieldValues = fields ? _.split(fields as string, ",") : []; const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())); if (!_.isEmpty(invalidFields)) { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS", `The specified fields [${invalidFields}] in the dataset cannot be found.`, "BAD_REQUEST", 400); diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index ca0ec0d1..2c05ce87 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -5,13 +5,17 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from "mocha"; import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; +import { apiId, defaultFields } from "../../../controllers/DatasetRead/DatasetRead"; import { TestInputsForDatasetRead } from "./Fixtures"; import { DatasetTransformations } from "../../../models/Transformation"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; import { ConnectorInstances } from "../../../models/ConnectorInstances"; +import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; +import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; +import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; chai.use(spies); chai.should(); @@ -134,7 +138,7 @@ describe("DATASET READ API", () => { }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft&mode=edit") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -148,6 +152,50 @@ describe("DATASET READ API", () => { }); }); + it("Dataset read success: Migrating v1 draft dataset to v2 on mode=edit", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.DRAFT_SCHEMA_V1) + }) + chai.spy.on(DatasetTransformationsDraft, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfigDraft, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) + }) + chai.spy.on(DatasetTransformationsDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfigDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasourceDraft, "destroy", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.name.should.be.eq("sb-telemetry") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(_.pick(TestInputsForDatasetRead.DRAFT_SCHEMA_V1, defaultFields))) + done(); + }); + }); + it("Dataset read failure: Updating dataset status to draft on mode=edit fails as live record not found", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts index 61760c6e..c1ecd1e2 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -24,6 +24,49 @@ export const TestInputsForDatasetRead = { ] } }, + DRAFT_SCHEMA_V1: { + "dataset_id": "sb-telemetry", + "name": "sb-telemetry", + "type": "event", + "status": "Draft", + "tags": [ + "tag1", + "tag2" + ], + "validation_config": { + "validate": true, + "validation_mode": "Strict", + "mode": "Strict" + }, + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "eid": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ets": { + "type": "string" + }, + "required": [ + "eid" + ] + }, + "additionalProperties": true + }, + "version": 1, + "api_version": "v1", + "dataset_config": { + "timestamp_key": "ets", + "data_key": "", + "redis_db_host": "localhost", + "redis_db_port": 6379, + "redis_db": 0 + } + }, LIVE_SCHEMA: { "dataset_id": "sb-telemetry", "name": "sb-telemetry", @@ -49,7 +92,7 @@ export const TestInputsForDatasetRead = { ] } }, - TRANSFORMATIONS_SCHEMA:[{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + TRANSFORMATIONS_SCHEMA: [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], TRANSFORMATIONS_SCHEMA_V1: [ { "field_key": "eid", @@ -67,27 +110,14 @@ export const TestInputsForDatasetRead = { } } ], - DATASOURCE_SCHEMA: { - "id": "sb-telemetry_sb-telemetry", - "datasource": "sb-telemetry", - "dataset_id": "sb-telemetry", - "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "dataset-conf_day", "dimensionsSpec": { "dimensions": [{ "type": "string", "name": "a" }, { "type": "string", "name": "obsrv.meta.source.connector" }, { "type": "string", "name": "obsrv.meta.source.id" }] }, "timestampSpec": { "column": "obsrv_meta.syncts", "format": "auto" }, "metricsSpec": [], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "none", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 5000000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "consumerProperties": { "bootstrap.servers": "localhost:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [{ "type": "path", "expr": "$.['a']", "name": "a" }, { "type": "path", "expr": "$.obsrv_meta.['syncts']", "name": "obsrv_meta.syncts" }, { "type": "path", "expr": "$.obsrv_meta.source.['connector']", "name": "obsrv.meta.source.connector" }, { "type": "path", "expr": "$.obsrv_meta.source.['connectorInstance']", "name": "obsrv.meta.source.id" }, { "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts", "type": "path" }] } }, "appendToExisting": false } } }, - "datasource_ref": "sb-telemetry_DAY", - "retention_period": { - "enabled": "false" - }, - "archival_policy": { - "enabled": "false" - }, - "purge_policy": { - "enabled": "false" - }, - "backup_config": { - "enabled": "false" - }, - "status": "Live", - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "published_date": "2023-07-03 00:00:00" - } + CONNECTORS_SCHEMA_V1: [ + { + "id": "hsh882ehdshe", + "connector_type": "kafka", + "connector_config": { + "topic": "local.ingest", + "brokerURL": "localhost:9092" + } + } + ] } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index c82e809d..2ce2fdfd 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -29,12 +29,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({}) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .post("/v2/datasets/status-transition") @@ -97,12 +92,6 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(TestInputsForDatasetStatusTransition.INVALID_SCHEMA_FOR_READY_TO_PUBLISH) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) chai .request(app) .post("/v2/datasets/status-transition") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts index 34907197..1122aac4 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts @@ -8,7 +8,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -29,12 +28,6 @@ describe("DATASET CONNECTORS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +54,6 @@ describe("DATASET CONNECTORS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index e8f614c8..eaa80648 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -8,7 +8,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; import { apiId, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -29,12 +28,6 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +54,6 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index cdf02c4b..afc673ca 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -8,7 +8,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -29,12 +28,6 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +54,6 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -99,12 +86,6 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 24ccc831..d17f1c3b 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -29,12 +29,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts index fe3715d4..49ba7b8a 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -29,12 +29,7 @@ describe("DATASET TAGS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET TAGS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index 0ae345ff..58252e83 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -29,12 +29,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -91,12 +81,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 1818bc79..6cb84e25 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -29,12 +29,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -66,12 +61,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index e77ab8b0..d5feed17 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -29,12 +29,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") From 13c499e57a1eefb9f498a40f98dbf17f592adaea Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 23:46:50 +0530 Subject: [PATCH 069/311] #OBS-I146: fix: status transition test cases --- .../DatasetCreate/DatasetCreate.spec.ts | 3 - .../DatasetRead/DatasetRead.spec.ts | 4 +- .../DatasetManagement/DatasetRead/Fixtures.ts | 10 ++ .../DatasetLive.spec.ts | 109 ++++++++++++++++++ .../DatasetReadyToPublish.spec.ts | 29 ++++- .../DatasetStatusTransition/Fixtures.ts | 42 ++++++- 6 files changed, 188 insertions(+), 9 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index a0e2ace6..51953ba0 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -30,9 +30,6 @@ describe("DATASET CREATE API", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(null) }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([{ nextVal: 9 }]) - }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: { id: "telemetry" } }) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 2c05ce87..6191d43e 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -99,7 +99,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) chai.spy.on(ConnectorInstances, "findAll", () => { - return Promise.resolve([]) + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) @@ -131,7 +131,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) }) chai.spy.on(DatasetSourceConfig, "findAll", () => { - return Promise.resolve([]) + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts index c1ecd1e2..2efaec73 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -119,5 +119,15 @@ export const TestInputsForDatasetRead = { "brokerURL": "localhost:9092" } } + ], + CONNECTORS_SCHEMA_V2: [ + { + "id": "hsh882ehdshe", + "connector_id": "kafka", + "connector_config": { + "topic": "local.ingest", + "brokerURL": "localhost:9092" + } + } ] } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index bb0c8e51..cd32573f 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -66,6 +66,115 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); + + it("Dataset status transition success: When the action is to set master dataset live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([[{ nextval: 9 }]]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("master-telemetry") + done(); + }); + }); + + it("Dataset status transition failure: When the dependent denorm master dataset is not live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(_.clone(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH)) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Retired", "dataset_config": { "redis_db": 21 }, "api_version": "v1" }]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.PRECONDITION_REQUIRED); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("The datasets with id:master-dataset are not in published status") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_LIVE") + done(); + }); + }); + + it("Dataset status transition failure: When dataset to publish is self referencing the denorm master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({...TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH, "id": "master-dataset"}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.CONFLICT); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("The denorm master dataset is self-referencing itself") + res.body.error.code.should.be.eq("SELF_REFERENCING_MASTER_DATA") + done(); + }); + }); + + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") + res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") + done(); + }); + }); + it("Dataset status transition failure: When dataset is not found to publish", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 2ce2fdfd..f7d5f1f3 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -7,8 +7,6 @@ import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; -import { sequelize } from "../../../connections/databaseConnection"; - chai.use(spies); chai.should(); @@ -29,7 +27,32 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({}) }) - + + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is make master dataset ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_MASTER_SCHEMA_FOR_READY_TO_PUBLISH) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai .request(app) .post("/v2/datasets/status-transition") diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index 9a807e7f..ecf531f0 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -23,6 +23,18 @@ export const TestInputsForDatasetStatusTransition = { "status": "Live" } }, + VALID_SCHEMA_FOR_LIVE_MASTER: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "master-telemetry", + "status": "Live" + } + }, VALID_SCHEMA_FOR_RETIRE: { "id": "api.datasets.status-transition", "ver": "v2", @@ -85,6 +97,32 @@ export const TestInputsForDatasetStatusTransition = { "sample_data": {}, "entry_topic": "local.ingest" }, + VALID_MASTER_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "master", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "data_key": "eid", "partition_key": "eid", "timestamp_key": "obsrv_meta.syncts" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": [] }, + "tags": ["tag1"], + "status": "Draft", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-07-24 19:12:13.021", + "updated_date": "2024-07-25 06:12:38.412", + "version_key": "1721887933020", + "transformations_config": [], + "connectors_config": [], + "api_version": "v2", + "sample_data": {}, + "entry_topic": "local.ingest" + }, INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { "id": "dataset-all-fields7", "dataset_id": "dataset-all-fields7", @@ -119,5 +157,7 @@ export const TestInputsForDatasetStatusTransition = { "version_key": "1721887933020", "api_version": "v2" }, - DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } + DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "master-telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_MASTER_DATASET_INVALID: { "dataset_id": "master-telemetry", "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From 9c8d4ac7d319cfba6e5918af7de19275722fe6f9 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 23:49:29 +0530 Subject: [PATCH 070/311] #OBS-I146: fix: Test case script fix --- api-service/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/package.json b/api-service/package.json index 1360ecf6..1d49c029 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "ts-node ./src/app.ts", "test": "source .env.test && nyc mocha ./src/tests/**/*.spec.ts --exit", - "actions:test": "nyc mocha ./src/test/*.spec.ts --exit", + "actions:test": "nyc mocha ./src/tests/**/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", "package": "npm run build && cd dist && npm pack . && cd ..", "lint": "eslint . --ext .ts", From 2079680dad5b4757f171d5931e7d4fe6fb0a1f2a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 23:54:58 +0530 Subject: [PATCH 071/311] #OBS-I146: fix: Type error fix --- api-service/src/services/CloudServices/AWSStorageService.ts | 2 +- api-service/src/services/CloudServices/AzureStorageService.ts | 2 +- api-service/src/services/CloudServices/GCPStorageService.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts index e538298f..d83b65b1 100644 --- a/api-service/src/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -67,7 +67,7 @@ export class AWSStorageService implements ICloudService { async getSignedUrls(container: any, filesList: any) { const signedUrlsPromises = this.generateSignedURLs(container, filesList) const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; const files: any[] = []; // Formatting response signedUrlsList.map(async (fileObject) => { diff --git a/api-service/src/services/CloudServices/AzureStorageService.ts b/api-service/src/services/CloudServices/AzureStorageService.ts index c90af51e..b4472aac 100644 --- a/api-service/src/services/CloudServices/AzureStorageService.ts +++ b/api-service/src/services/CloudServices/AzureStorageService.ts @@ -130,7 +130,7 @@ export class AzureStorageService implements ICloudService { const signedUrlsPromises = this.generateSignedURLs(container, filesList) const signedUrlsList = await Promise.all(signedUrlsPromises); const files: any[] = [] - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; // Formatting response signedUrlsList.map(async (fileObject) => { const fileDetails = _.keys(fileObject); diff --git a/api-service/src/services/CloudServices/GCPStorageService.ts b/api-service/src/services/CloudServices/GCPStorageService.ts index 90fcaeb0..b07e3e4e 100644 --- a/api-service/src/services/CloudServices/GCPStorageService.ts +++ b/api-service/src/services/CloudServices/GCPStorageService.ts @@ -74,7 +74,7 @@ export class GCPStorageService implements ICloudService { return generateSignedUrls() .then(signedUrlList => { - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; const files: any = []; const signedUrls = _.flattenDeep(_.map(signedUrlList, url => { const values = _.values(url) From 55737187add2a69245fdfa8176ac9b50953f4d3b Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 21 Aug 2024 10:33:49 +0530 Subject: [PATCH 072/311] #OBS-I141: removed metric for sum of response time --- .../DatasetCopy/DatasetCopyHelper.ts | 1 - .../src/controllers/DatasetRead/DatasetRead.ts | 17 +++++++++++++---- api-service/src/services/DatasetService.ts | 16 ++++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index de4c8087..82b35ebd 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -10,7 +10,6 @@ export const updateRecords = (datasetRecord: Record, newDatasetId: _.set(datasetRecord, 'status', DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) - _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) _.set(datasetRecord, 'version', version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 28909ff3..db2dc32f 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -39,10 +39,19 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ - ...connector, - connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) - })); + dataset.connectors_config = dataset?.connectors_config.map((connector: any) => { + let connector_config = _.get(connector, "connector_config") + const authMechanism = _.get(connector_config, ["authenticationMechanism"]) + if (authMechanism && authMechanism.encrypted) { + connector_config = { + ...connector_config, + authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues))} + } + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) + } + }); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 25b120f2..4f39b5d1 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -125,12 +125,14 @@ class DatasetService { const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, ["field_key"]), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section) }, mode: _.get(config, ["mode"]) } @@ -147,6 +149,7 @@ class DatasetService { }) draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["status"] = DatasetStatus.Draft return draftDataset; } @@ -157,6 +160,8 @@ class DatasetService { return "pii"; case "additionalFields": return "derived"; + case "derived": + return "derived"; default: return "transform"; } @@ -184,12 +189,14 @@ class DatasetService { }) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, "field_key"), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, "metadata._transformedFieldDataType") || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section), }, mode: _.get(config, "mode") } @@ -225,8 +232,9 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft - await DatasetDraft.create(draftDataset); - return await this.getDraftDataset(draftDataset.dataset_id); + const response = await DatasetDraft.create(draftDataset); + return _.get(response,"dataValues"); + // return await this.getDraftDataset(draftDataset.dataset_id); } getNextRedisDBIndex = async () => { From 4429b3362d6281e30d898b70567de1c4add3f83d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 21 Aug 2024 11:01:33 +0530 Subject: [PATCH 073/311] #OBS-I146: fix: Dataset read api test cases fixes --- api-service/src/services/DatasourceService.ts | 338 ------------------ .../DatasetRead/DatasetRead.spec.ts | 66 ++++ .../DatasetManagement/DatasetRead/Fixtures.ts | 10 + .../DatasetLive.spec.ts | 45 ++- 4 files changed, 98 insertions(+), 361 deletions(-) diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index 96c8f294..f9f6b2a7 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,37 +1,5 @@ import _ from "lodash"; -import { ingestionConfig } from "../configs/IngestionConfig"; import { Datasource } from "../models/Datasource"; -import { DatasourceDraft } from "../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../models/TransformationDraft"; -import { DatasetTransformations } from "../models/Transformation"; -import { DatasetStatus } from "../types/DatasetModels"; -import { Dataset } from "../models/Dataset"; - -export const DEFAULT_TIMESTAMP = { - indexValue: "obsrv_meta.syncts", - rootPath: "obsrv_meta", - label: "syncts", - path: "obsrv_meta.properties.syncts", -} - -const defaultTsObject = [ - { - "column": DEFAULT_TIMESTAMP.rootPath, - "type": "object", - "key": `properties.${DEFAULT_TIMESTAMP.rootPath}`, - "ref": `properties.${DEFAULT_TIMESTAMP.rootPath}`, - "isModified": true, - "required": false, - }, - { - "column": DEFAULT_TIMESTAMP.label, - "type": "integer", - "key": `properties.${DEFAULT_TIMESTAMP.path}`, - "ref": `properties.${DEFAULT_TIMESTAMP.path}`, - "isModified": true, - "required": false, - } -] export const getDatasourceList = async (datasetId: string, raw = false) => { const dataSource = await Datasource.findAll({ @@ -43,320 +11,14 @@ export const getDatasourceList = async (datasetId: string, raw = false) => { return dataSource } -export const getDraftDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await DatasourceDraft.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - -export const getDatasource = async (datasetId: string) => { - const dataSource = await Datasource.findOne({ - where: { - dataset_id: datasetId, - }, - }); - return dataSource -} - -export const getUpdatedSchema = async (configs: Record) => { - const { id, transformation_config, denorm_config, data_schema, action, indexCol = ingestionConfig.indexCol["Event Arrival Time"] } = configs - const existingTransformations = await DatasetTransformationsDraft.findAll({ where: { dataset_id: id }, raw: true }) - let resultantTransformations: any[] = [] - if (action === "edit") { - const toDeleteTransformations = _.compact(_.map(transformation_config, config => { - if (_.includes(["update", "remove"], _.get(config, "action"))) { - return _.get(config, ["value", "field_key"]) - } - })) - const updatedExistingTransformations = _.compact(_.map(existingTransformations, configs => { - if (!_.includes(toDeleteTransformations, _.get(configs, "field_key"))) { - return configs - } - })) || [] - const newTransformations = _.compact(_.map(transformation_config, config => { - if (_.includes(["update", "add"], _.get(config, "action"))) { - return config - } - })) || [] - resultantTransformations = [...updatedExistingTransformations, ...newTransformations] - } - if (action === "create") { - resultantTransformations = transformation_config || [] - } - let denormFields = _.get(denorm_config, "denorm_fields") - let updatedColumns = flattenSchema(data_schema) - const transformedFields = _.filter(resultantTransformations, field => _.get(field, ["metadata", "section"]) === "transformation") - let additionalFields = _.filter(resultantTransformations, field => _.get(field, ["metadata", "section"]) === "additionalFields") - updatedColumns = _.map(updatedColumns, (item) => { - const transformedData = _.find(transformedFields, { field_key: item.column }); - if (transformedData) { - const data = _.get(transformedData, "metadata") - return { - ...item, - type: _.get(data, "_transformedFieldSchemaType") || "string", - isModified: true, - ...data - }; - } - return item; - }); - denormFields = _.size(denormFields) ? await formatDenormFields(denormFields) : [] - additionalFields = formatNewFields(additionalFields, null); - let ingestionPayload = { schema: [...updatedColumns, ...denormFields, ...additionalFields] }; - if (indexCol === ingestionConfig.indexCol["Event Arrival Time"]) - ingestionPayload = { schema: [...updatedColumns, ...defaultTsObject, ...denormFields, ...additionalFields] }; - const updatedIngestionPayload = updateJSONSchema(data_schema, ingestionPayload) - return updatedIngestionPayload -} - -export const updateJSONSchema = (schema: Record, updatePayload: Record) => { - const clonedOriginal = _.cloneDeep(schema); - const modifiedRows = _.filter(_.get(updatePayload, "schema"), ["isModified", true]); - _.forEach(modifiedRows, modifiedRow => { - const { isDeleted = false, required = false, key, type, description = null, arrival_format, data_type, isModified = false } = modifiedRow; - if (isDeleted) { - deleteItemFromSchema(clonedOriginal, `${key}`, false); - } else { - updateTypeInSchema(clonedOriginal, `${key}`, type, true); - updateFormatInSchema(clonedOriginal, `${key}`, arrival_format); - updateDataTypeInSchema(clonedOriginal, `${key}`, data_type, isModified); - descriptionInSchema(clonedOriginal, `${key}`, description); - changeRequiredPropertyInSchema(clonedOriginal, `${key}`, required); - } - }); - return clonedOriginal; -} - -const updateDataTypeInSchema = (schema: Record, schemaPath: string, data_type: string, isModified: boolean) => { - const existing = _.get(schema, schemaPath); - if (isModified) { - const validDateFormats = ["date-time", "date", "epoch"] - if (!_.includes(validDateFormats, data_type)) { - _.unset(existing, "format"); - } else { - data_type === "epoch" ? _.set(existing, "format", "date-time") : _.set(existing, "format", data_type) - } - } - _.set(schema, schemaPath, { ...existing, data_type }); -} - - -const descriptionInSchema = (schema: Record, schemaPath: string, description: string) => { - const existing = _.get(schema, schemaPath); - if (description) _.set(schema, schemaPath, { ...existing, description }); -} - -const updateFormatInSchema = (schema: Record, schemaPath: string, arrival_format: string) => { - const existing = _.get(schema, schemaPath); - _.set(schema, schemaPath, { ...existing, arrival_format }); -} - -const updateTypeInSchema = (schema: Record, schemaPath: string, type: string, removeSuggestions: boolean = false) => { - const existing = _.get(schema, schemaPath); - if (removeSuggestions) { - _.unset(existing, "suggestions"); - _.unset(existing, "oneof"); - _.unset(existing, "arrivalOneOf") - } - _.set(schema, schemaPath, { ...existing, type }); -} - - -const deleteItemFromSchema = (schema: Record, schemaKeyPath: string, required: boolean) => { - if (_.has(schema, schemaKeyPath)) { - _.unset(schema, schemaKeyPath); - changeRequiredPropertyInSchema(schema, schemaKeyPath, required); - } -} - -const getPathToRequiredKey = (schema: Record, schemaKeyPath: string, schemaKey: string) => { - const regExStr = `properties.${schemaKey}`; - const regex = `(.${regExStr})`; - const [pathToRequiredKey] = _.split(schemaKeyPath, new RegExp(regex, "g")); - if (pathToRequiredKey === schemaKeyPath) return "required" - return `${pathToRequiredKey}.required` -} - -const changeRequiredPropertyInSchema = (schema: Record, schemaKeyPath: string, required: boolean) => { - const schemaKey = _.last(_.split(schemaKeyPath, ".")); - if (schemaKey) { - const pathToRequiredProperty = getPathToRequiredKey(schema, schemaKeyPath, schemaKey); - const existingRequiredKeys = _.get(schema, pathToRequiredProperty) || []; - if (required) { - // add to required property. - const updatedRequiredKeys = _.includes(existingRequiredKeys, schemaKey) ? existingRequiredKeys : [...existingRequiredKeys, schemaKey]; - _.set(schema, pathToRequiredProperty, updatedRequiredKeys); - } else { - // remove from required property. - const updatedRequiredKeys = _.difference(existingRequiredKeys, [schemaKey]); - if (_.size(updatedRequiredKeys) > 0) - _.set(schema, pathToRequiredProperty, updatedRequiredKeys); - } - } -} -export const formatNewFields = (newFields: Record, dataMappings: any) => { - if (newFields.length > 0) { - const final = _.map(newFields, (item: any) => { - const columnKey = _.join(_.map(_.split(_.get(item, "field_key"), "."), payload => `properties.${payload}`), ".") - return { - "column": item.field_key, - "type": _.get(item, ["metadata", "_transformedFieldSchemaType"]) || "string", - "key": columnKey, - "ref": columnKey, - "isModified": true, - "required": false, - "data_type": _.get(item, ["metadata", "_transformedFieldDataType"]), - ...(dataMappings && { "arrival_format": getArrivalFormat(_.get(item, "_transformedFieldSchemaType"), dataMappings) || _.get(item, "arrival_format") }) - } - }); - return final; - } - else return []; -} -const getArrivalFormat = (data_type: string | undefined, dataMappings: Record) => { - let result = null; - if (data_type) { - _.forEach(dataMappings, (value, key) => { - if (_.includes(_.get(value, "arrival_format"), data_type)) { - result = key; - } - }); - } - return result; -} -export const updateDenormDerived = (schemaColumns: any, columns: any, fixedPrefix: string): any[] => { - const result = _.map(columns, (column: any) => { - const isExistingColumn = _.find(schemaColumns, ["column", column.field_key]); - if (isExistingColumn) { - return { - ...isExistingColumn, - "type": _.get(column, "metadata._transformedFieldSchemaType"), - "data_type": _.get(column, "metadata._transformedFieldDataType"), - "required": false, - "isModified": true, - ..._.get(column, "metadata"), - }; - } else { - const columnKey = _.join(_.map(_.split(_.get(column, "field_key"), "."), payload => `properties.${payload}`), ".") - return { - "column": `${fixedPrefix}.${column.field_key}`, - "type": _.get(column, "metadata._transformedFieldSchemaType"), - "key": `properties.${fixedPrefix}.${columnKey}`, - "ref": `properties.${fixedPrefix}.${columnKey}`, - "required": false, - "isModified": true, - "data_type": _.get(column, "metadata._transformedFieldDataType"), - ..._.get(column, "metadata"), - }; - } - }); - return _.concat(schemaColumns, result); -} -const processDenormConfigurations = async (item: any) => { - const denormFieldsData: any = []; - const redis_db = _.get(item, "redis_db"); - const denorm_out_field = _.get(item, "denorm_out_field"); - const dataset: any = await Dataset.findOne({ where: { "dataset_config": { "redis_db": redis_db } }, raw: true }) || [] - const transformations = _.size(dataset) ? await DatasetTransformations.findAll({ where: { status: DatasetStatus.Live, dataset_id: _.get(dataset, "dataset_id") }, raw: true }) : [] - let schema = flattenSchema(_.get(dataset, "data_schema"), denorm_out_field, true); - schema = updateDenormDerived(schema, _.get(transformations, "data.result"), denorm_out_field,); - denormFieldsData.push({ - "column": denorm_out_field, - "type": "object", - "key": `properties.${denorm_out_field}`, - "ref": `properties.${denorm_out_field}`, - "isModified": true, - "required": false, - "arrival_format": "object", - "data_type": "object", - }); - denormFieldsData.push(...schema); - return denormFieldsData; -} -export const formatDenormFields = async (denormFields: any) => { - if (denormFields.length > 0) { - const final = _.map(denormFields, (item: any) => { - return processDenormConfigurations(item); - }); - return Promise.all(final).then((data) => _.flatten(data)); - } - else return []; -} -const addRequiredFields = ( - type: string, - result: Record, - schemaObject: Record, - required: string[], -) => { - const requiredFields = _.get(schemaObject, "required") || []; - _.map(result, (item) => { - if (type === "array" || type === "object") { - if (required && required.includes(item.key.replace("properties.", ""))) item.required = true; - else if (requiredFields.includes(item.key.replace("properties.", ""))) item.required = true; - else item.required = false; - } - else if (requiredFields.includes(item.key.replace("properties.", ""))) item.required = true; - else item.required = false; - }) -} -const flatten = (schemaObject: Record, rollup: boolean = false) => { - const schemaObjectData = schemaObject; - const result: Record = {}; - const getKeyName = (prefix: string, key: string) => prefix ? `${prefix}.${key}` : key; - const flattenHelperFn = (propertySchema: Record, prefix: string, ref: string, arrayChild = false) => { - const { type, properties, items, required = false, ...rest } = propertySchema || {}; - if (type === "object" && properties) { - if (prefix !== "" && !arrayChild) result[prefix] = { type, key: ref, ref, properties, items, parent: true, ...rest }; - for (const [key, value] of Object.entries(properties)) { - flattenHelperFn(value as Record, getKeyName(prefix, key), getKeyName(ref, `properties.${key}`)); - } - } else if (type === "array" && items && !rollup) { - if (prefix !== "") result[prefix] = { type, key: ref, ref, properties, items, parent: true, ...rest }; - if (["array", "object"].includes(items?.type)) { - flattenHelperFn(items, prefix, getKeyName(ref, `items`), true) - } else { - result[prefix] = { type, key: ref, ref, properties, items, ...rest }; - } - } else { - result[prefix] = { type, key: ref, ref, properties, items, ...rest }; - } - addRequiredFields(type, result, schemaObjectData, required); - } - flattenHelperFn(schemaObjectData, "", ""); - return result; -} -export const flattenSchema = (schema: Record, fixedPrefix?: string | undefined, modified?: boolean, rollup: boolean = false) => { - const flattend = flatten(schema, rollup); - if (fixedPrefix) - return _.map(flattend, (value, key) => { - const { key: propertyKey, ref } = value; - const keySplit = _.split(propertyKey, "."); - const refSplit = _.split(ref, "."); - keySplit.splice(1, 0, fixedPrefix, "properties"); - refSplit.splice(1, 0, fixedPrefix, "properties"); - const data = { - column: `${fixedPrefix}.${key}`, - ...value, - key: keySplit.join("."), - ref: refSplit.join("."), - }; - if (modified) { data.isModified = true; data.required = false; } - return data; - }); - return _.map(flattend, (value, key) => ({ column: key, ...value })); -} \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 6191d43e..cfd333eb 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -101,6 +101,9 @@ describe("DATASET READ API", () => { chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.MASTER_DATASET_SCHEMA) + }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) @@ -133,6 +136,9 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetSourceConfig, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.MASTER_DATASET_SCHEMA) + }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) @@ -217,6 +223,66 @@ describe("DATASET READ API", () => { }); }); + it("Dataset read failure: When dependent denorm master dataset not found", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("The dependent dataset not found") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_FOUND") + done(); + }); + }); + + it("Dataset read failure: When dependent denorm master dataset not live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":20}}}]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.PRECONDITION_REQUIRED); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("The dependent master dataset is not published") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_LIVE") + done(); + }); + }); + it("Dataset read failure: When the dataset of requested dataset_id not found", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(null) diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts index 2efaec73..d520a102 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -78,6 +78,15 @@ export const TestInputsForDatasetRead = { ], "data_version": 1, "api_version": "v2", + "denorm_config": { + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "redis_db": 16 + } + ] + }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, @@ -92,6 +101,7 @@ export const TestInputsForDatasetRead = { ] } }, + MASTER_DATASET_SCHEMA:[{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":16}}}], TRANSFORMATIONS_SCHEMA: [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], TRANSFORMATIONS_SCHEMA_V1: [ { diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index cd32573f..59598719 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -66,6 +66,28 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") + res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") + done(); + }); + }); it("Dataset status transition success: When the action is to set master dataset live", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { @@ -152,29 +174,6 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); - it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) - }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq("api.datasets.status-transition"); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") - res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") - done(); - }); - }); - it("Dataset status transition failure: When dataset is not found to publish", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() From f07caf5c5ccd02ea0c84c244c06c27cc16d896a8 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 21 Aug 2024 11:46:55 +0530 Subject: [PATCH 074/311] #OBS-I146: fix: Hudi spec generation test cases --- api-service/src/services/DatasourceService.ts | 1 - .../DatasetLive.spec.ts | 88 +++++++++++++++++++ .../DatasetStatusTransition/Fixtures.ts | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index f9f6b2a7..812a5611 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,4 +1,3 @@ -import _ from "lodash"; import { Datasource } from "../models/Datasource"; export const getDatasourceList = async (datasetId: string, raw = false) => { diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 59598719..7c787387 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -11,6 +11,7 @@ import { commandHttpService } from "../../../connections/commandServiceConnectio import { sequelize } from "../../../connections/databaseConnection"; import { DatasourceDraft } from "../../../models/DatasourceDraft"; import { Dataset } from "../../../models/Dataset"; +import { Datasource } from "../../../models/Datasource"; chai.use(spies); chai.should(); @@ -66,6 +67,93 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); + it("Dataset status transition success: When the action is to set dataset live v1 by creating hudi spec", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is to set dataset live v2 by updating hudi spec", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "api_version":"v2", "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "findOne", () => { + return Promise.resolve({"ingestion_spec":{"dataset": "dataset-all-fields4", "schema": {"table": "dataset-all-fields4_events", "partitionColumn": "eid", "timestampColumn": "obsrv_meta.syncts", "primaryKey": "eid", "columnSpec": [{"type": "string", "name": "mid", "index": 1}, {"type": "epoch", "name": "ets", "index": 2}, {"type": "string", "name": "userdata.mid", "index": 3}, {"type": "epoch", "name": "userdata.ets", "index": 4}, {"type": "string", "name": "userdata.eid", "index": 5}, {"type": "string", "name": "email", "index": 6}, {"type": "string", "name": "obsrv.meta.source.connector", "index": 7}, {"type": "string", "name": "obsrv.meta.source.id", "index": 8}]}, "inputFormat": {"type": "json", "flattenSpec": {"fields": [{"type": "path", "expr": "$.mid", "name": "mid"}, {"type": "path", "expr": "$.ets", "name": "ets"}, {"type": "path", "expr": "$.eid", "name": "eid"}, {"type": "path", "expr": "$.userdata.mid", "name": "userdata.mid"}, {"type": "path", "expr": "$.userdata.ets", "name": "userdata.ets"}, {"type": "path", "expr": "$.userdata.eid", "name": "userdata.eid"}, {"type": "path", "expr": "$.email", "name": "email"}, {"type": "path", "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts"}, {"type": "path", "expr": "$.obsrv_meta.source.connector", "name": "obsrv.meta.source.connector"}, {"type": "path", "expr": "$.obsrv_meta.source.connectorInstance", "name": "obsrv.meta.source.id"}]}}}}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index ecf531f0..e65f1852 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -158,6 +158,7 @@ export const TestInputsForDatasetStatusTransition = { "api_version": "v2" }, DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "ets", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "master-telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, DRAFT_MASTER_DATASET_INVALID: { "dataset_id": "master-telemetry", "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From 802eb731c76ad41620d876c54545b29c823d4537 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 21 Aug 2024 23:31:09 +0530 Subject: [PATCH 075/311] #OBS-I146: fix: Test case and linting fix --- api-service/.eslintrc | 3 +- api-service/src/middlewares/errors.ts | 4 +- .../DatasetLive.spec.ts | 6 -- .../DatasetRetire.spec.ts | 6 -- .../DatasetUpdate/DatasetDenorm.spec.ts | 32 ++++++++++ .../DatasetUpdate/DatasetUpdate.spec.ts | 62 +++++++++---------- .../DatasetUpdate/Fixtures.ts | 23 ++++++- 7 files changed, 89 insertions(+), 47 deletions(-) diff --git a/api-service/.eslintrc b/api-service/.eslintrc index 598451ab..b209a5f4 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -9,7 +9,8 @@ "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], - "rules": { + "rules": { + "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], "@typescript-eslint/quotes": [ diff --git a/api-service/src/middlewares/errors.ts b/api-service/src/middlewares/errors.ts index 96c8069e..ce3200ab 100644 --- a/api-service/src/middlewares/errors.ts +++ b/api-service/src/middlewares/errors.ts @@ -4,7 +4,7 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; -export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { +export const errorHandler = (err: Error, req: Request, res: Response, _next: NextFunction) => { logger.error({ path: req.url, req: req.body , ...err }) const errorMessage = {name: err.name, message: err.message}; @@ -12,7 +12,7 @@ export const errorHandler = (err: Error, req: Request, res: Response, next: Next }; -export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, _next: NextFunction) => { logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 7c787387..e26ddafe 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -187,12 +187,6 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({}) }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) - }) - chai.spy.on(DatasourceDraft, "create", () => { - return Promise.resolve({}) - }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 0be1fdb9..1a0428af 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -91,12 +91,6 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve() - }) chai.spy.on(commandHttpService, "post", () => { return Promise.resolve({}) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index afc673ca..5f330032 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -103,4 +103,36 @@ describe("DATASET DENORM UPDATE", () => { }); }); + it("Success: Ignore the denorm when payload contains denorm that already exists", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", version_key: validVersionKey, type:"dataset", api_version:"v2", status: "Draft", denorm_config: { + denorm_fields: [ { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }] + } + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_EXISTING_DENORM) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + }) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 6cb84e25..c2050f23 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -29,7 +29,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -61,7 +61,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -80,7 +80,7 @@ describe("DATASET UPDATE API", () => { }); it("Dataset updation failure: When no fields with dataset_id is provided in the request payload", (done) => { - + chai .request(app) .patch("/v2/datasets/update") @@ -101,7 +101,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -120,9 +120,9 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: When dataset to update is outdated", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", type:"event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", type: "event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -139,11 +139,32 @@ describe("DATASET UPDATE API", () => { }); }); + it("Dataset updation failure: When dataset to update is of api_version v1", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ id: "telemetry", type: "event", status: "Draft", version_key: "1813444815918", api_version: "v1" }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, name: "telemetry" } }) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset") + res.body.error.code.should.be.eq("DATASET_API_VERSION_MISMATCH") + done(); + }); + }); + it("Dataset updation failure: Dataset to update is not in draft state", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", type:"event", status: "Live", version_key: "1713444815918", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", type: "event", status: "Live", version_key: "1713444815918", api_version: "v2" }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -187,17 +208,11 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset name updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -216,7 +231,7 @@ describe("DATASET UPDATE API", () => { }); it("Failure: Failed to update the dataset name", (done) => { - + chai .request(app) .patch("/v2/datasets/update") @@ -246,18 +261,9 @@ describe("DATASET UPDATE API", () => { id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -308,12 +314,6 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 97248d9f..ae9628a4 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -15,7 +15,8 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "name": "telemetry" + "name": "telemetry", + "sample_data":{"events":{}} } }, @@ -383,6 +384,26 @@ export const TestInputsForDatasetUpdate = { } }, + DATASET_UPDATE_WITH_EXISTING_DENORM: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "name": "sb-telemetry", + "denorm_config": { + "denorm_fields": [ + { + "value": { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }, + "action": "upsert" + } + ] + } + } + }, + DATASET_UPDATE_WITH_SAME_TRANSFORMATION_ADD_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", From 2b4740c077e01bf9370b7781a176c7c3615de68f Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 22 Aug 2024 15:57:11 +0530 Subject: [PATCH 076/311] merge commit --- .../DatasetCopy/DatasetCopyHelper.ts | 1 - .../RequestValidationSchemaV2.json | 3 + api-service/src/services/DatasetService.ts | 100 ++++++++++-------- 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index de4c8087..82b35ebd 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -10,7 +10,6 @@ export const updateRecords = (datasetRecord: Record, newDatasetId: _.set(datasetRecord, 'status', DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) - _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) _.set(datasetRecord, 'version', version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json index 45820502..da61770c 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -77,6 +77,9 @@ "type": "boolean", "default": true }, + "batch_id": { + "type": "string" + }, "extraction_key": { "type": "string", "default": "events" diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 25b120f2..0dfee5b6 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -25,7 +25,7 @@ class DatasetService { } findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Dataset.findAll({where, attributes, order, raw: true}) + return Dataset.findAll({ where, attributes, order, raw: true }) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -39,9 +39,9 @@ class DatasetService { } checkDatasetExists = async (dataset_id: string): Promise => { - const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); if (draft === null) { - const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); return !(live === null) } else { return true; @@ -53,7 +53,7 @@ class DatasetService { } findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return DatasetDraft.findAll({where, attributes, order, raw: true}) + return DatasetDraft.findAll({ where, attributes, order, raw: true }) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -78,7 +78,7 @@ class DatasetService { updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; @@ -96,7 +96,6 @@ class DatasetService { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -125,12 +124,14 @@ class DatasetService { const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, ["field_key"]), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section) }, mode: _.get(config, ["mode"]) } @@ -147,31 +148,34 @@ class DatasetService { }) draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["status"] = DatasetStatus.Draft return draftDataset; } - getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string): string => { - switch(section) { + switch (section) { case "pii": return "pii"; case "additionalFields": return "derived"; + case "derived": + return "derived"; default: return "transform"; } } createDraftDatasetFromLive = async (dataset: Model) => { - - let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); - const dataset_config:any = _.get(dataset, "dataset_config"); - const api_version:any = _.get(dataset, "api_version"); - if(api_version === "v1") { + + let draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { @@ -184,12 +188,14 @@ class DatasetService { }) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, "field_key"), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, "metadata._transformedFieldDataType") || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section), }, mode: _.get(config, "mode") } @@ -205,14 +211,14 @@ class DatasetService { } const denormConfig = _.get(draftDataset, "denorm_config") if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) if (_.isEmpty(masterDatasets)) { throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } } const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { const { redis_db, denorm_out_field, denorm_key } = field let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) - if(!masterConfig){ + if (!masterConfig) { masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) } if (_.isEmpty(masterConfig)) { @@ -238,12 +244,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -255,18 +261,18 @@ class DatasetService { try { await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); - } catch(err:any) { + } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Datasource.findAll({where, attributes, order, raw: true}) + return Datasource.findAll({ where, attributes, order, raw: true }) } private deleteDruidSupervisors = async (dataset: Record) => { @@ -290,26 +296,26 @@ class DatasetService { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - if(indexingConfig.olap_store_enabled) { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { await this.createDruidDataSource(draftDataset, transaction); } - if(indexingConfig.lakehouse_enabled) { + if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if(liveDataset && liveDataset.api_version === "v2") { + if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } await transaction.commit() - } catch(err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } await executeCommand(draftDataset.id, "PUBLISH_DATASET"); - + } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -318,7 +324,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -327,25 +333,25 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") - const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } - private createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string): Record => { - const datasource = _.join([draftDataset.dataset_id,"events"], "_") + const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { - id: _.join([datasource,type], '_'), + id: _.join([datasource, type], '_'), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, @@ -356,15 +362,15 @@ class DatasetService { } export const getLiveDatasetConfigs = async (dataset_id: string) => { - + let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - if(!_.isEmpty(transformations)){ + if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } - if(!_.isEmpty(connectors)){ + if (!_.isEmpty(connectors)) { datasetRecord["connectors_config"] = connectors } return datasetRecord; From 82be980889edd23427abf92551cc6888ba267c40 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 22 Aug 2024 16:02:42 +0530 Subject: [PATCH 077/311] #OBS-I173: fix: Dataset web console required fixes --- .../RequestValidationSchemaV2.json | 3 +++ .../ReadyToPublishSchema.json | 6 ++---- api-service/src/services/DatasetService.ts | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json index 45820502..da61770c 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -77,6 +77,9 @@ "type": "boolean", "default": true }, + "batch_id": { + "type": "string" + }, "extraction_key": { "type": "string", "default": "events" diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index f378a5f7..bbb67fdb 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -324,8 +324,7 @@ "minLength": 1 }, "expr": { - "type": "string", - "minLength": 1 + "type": "string" }, "condition": { "type": "object", @@ -335,8 +334,7 @@ "minLength": 1 }, "expr": { - "type": "string", - "minLength": 1 + "type": "string" } }, "required": ["type", "expr"], diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 28c1684e..71701e82 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -376,7 +376,9 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) - const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV1 = await getV1Connectors(dataset_id) + const connectors = _.concat(connectorsV1,connectorsV2) if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations @@ -387,4 +389,17 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { return datasetRecord; } +export const getV1Connectors = async (datasetId: string) => { + const v1connectors = await datasetService.getConnectorsV1(datasetId, ["id", "connector_type", "connector_config"]); + const modifiedV1Connectors = _.map(v1connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + return modifiedV1Connectors; +} + export const datasetService = new DatasetService(); \ No newline at end of file From 671a842dfd0f9630131a281d9eaae871f8f6d010 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 22 Aug 2024 18:31:58 +0530 Subject: [PATCH 078/311] #OBS-I173: fix: Dataset update changes to accept type changes --- api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts | 1 + .../DatasetUpdate/DatasetUpdateValidationSchema.json | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 25c9a633..590ea9b5 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -79,6 +79,7 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data + if(datasetReq.type) dataset["type"] = datasetReq.type return dataset; } diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 4dbc3e89..b31fe674 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -32,6 +32,10 @@ "version_key": { "type": "string" }, + "type": { + "type": "string", + "enum": ["event", "transaction", "master"] + }, "name": { "type": "string", "minLength": 1 @@ -199,7 +203,7 @@ } }, "additionalProperties": false - }, + }, "keys_config": { "type": "object", "properties": { From 35e47d565b86ab606d2e221fc14116db5355885d Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 23 Aug 2024 10:47:09 +0530 Subject: [PATCH 079/311] #OBS-I167 : dataset read api changes to read live dataset source configs --- .../src/controllers/DatasetRead/DatasetRead.ts | 17 +++++------------ .../DatasetStatusTransition.ts | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 2f6f3855..8bae2d9d 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -3,7 +3,7 @@ import httpStatus from "http-status"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetDraft } from "../../models/DatasetDraft"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, getV1Connectors } from "../../services/DatasetService"; import { obsrvError } from "../../types/ObsrvError"; import { cipherService } from "../../services/CipherService"; @@ -79,19 +79,10 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { - return { - id: _.get(config, "id"), - connector_id: _.get(config, "connector_type"), - connector_config: _.get(config, "connector_config"), - version: "v1" - } - }) + datasetConfigs["connectors_config"] = await getV1Connectors(datasetId) datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { - console.log(config) const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); return { field_key: _.get(config, "field_key"), @@ -105,7 +96,9 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise) => { let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) - defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + defaultConfigs = _.omit(defaultConfigs, ["router_config","dedupe_config"]) if (draftDataset?.type === 'master') { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } From 181de2b5d4a33d5fa3cd2706278c84185864972e Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 23 Aug 2024 10:57:29 +0530 Subject: [PATCH 080/311] #OBS-I146: fix: linting fix --- api-service/.eslintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/.eslintrc b/api-service/.eslintrc index b209a5f4..e7996aa9 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -9,7 +9,8 @@ "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], - "rules": { + "rules": { + "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], From 53262d19bc816130f1777acbc8e6c75978cfb364 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 23 Aug 2024 11:07:49 +0530 Subject: [PATCH 081/311] #OBS-I146: fix: linting fix --- api-service/.eslintrc | 1 - api-service/src/services/fs.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/.eslintrc b/api-service/.eslintrc index e7996aa9..ca799240 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -10,7 +10,6 @@ "plugin:@typescript-eslint/recommended" ], "rules": { - "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], diff --git a/api-service/src/services/fs.ts b/api-service/src/services/fs.ts index 774b87b9..cfc0010b 100644 --- a/api-service/src/services/fs.ts +++ b/api-service/src/services/fs.ts @@ -8,6 +8,7 @@ export const scrapModules = (folderPath: string, .map((file) => { const { default: { name, ...others }, + /* eslint-disable @typescript-eslint/no-var-requires */ } = require(path.join(folderPath, file)) as { default: Type }; mapping.set(name, others); From aed84c48a60b3d0bcd2944199a9eb16a526af31e Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 23 Aug 2024 11:15:44 +0530 Subject: [PATCH 082/311] #OBS-I167 : Added string or dict as type to connector_config --- command-service/src/command/dataset_command.py | 1 - command-service/src/model/db_models.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index db7afa7d..74a0a6fe 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -103,5 +103,4 @@ def audit_live_dataset(self, command_payload: CommandPayload, ts: int): "Failed to get dataset configurations from export API, dataset_id: ", dataset_id, ) - print("Dataset record details: ", json.dumps(dataset_record)) return False diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index f77f8314..23e37659 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -79,7 +79,7 @@ class DatasourcesDraft: class DatasetConnectorConfigDraft: id: str connector_id: str - connector_config: str | None + connector_config: str | dict version: str operations_config: dict | None = None data_format: str | None = 'json' From f0ba9239f1e74f93d5f25f0d5f8223e84c2ae1cf Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 23 Aug 2024 13:03:22 +0530 Subject: [PATCH 083/311] #OBS-I143: dataset publish changes fixes --- command-service/Dockerfile | 2 +- .../flink-connector/templates/deployment.yaml | 4 +- .../helm-charts/flink-connector/values.yaml | 9 +- .../spark-connector-cron/Chart.yaml | 2 +- .../src/command/connector_command.py | 324 ++++++++---------- 5 files changed, 154 insertions(+), 187 deletions(-) diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 2aa9a1b5..2ca4edef 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -1,7 +1,7 @@ FROM --platform=linux/amd64 python:3.12-alpine COPY --from=ubuntu /usr/local/bin /usr/local/bin -RUN apk update && apk add curl jq && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ +RUN apk update && apk add curl jq vim && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ RUN cd /tmp && curl -OL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ && tar -zxvf helm-v3.13.2-linux-amd64.tar.gz \ diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 946ca29d..6f4f98b7 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -181,8 +181,8 @@ spec: sleep 30s; /opt/flink/bin/flink run -m \ {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ - /flink/connectors/{{ $jobName }}-1.0.0/{{ $jobName }}-1.0.0.jar \ - --config.file.path /data/flink/conf/flink-connector.conf \ + /flink/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ + --config.file.path /data/flink/conf/connectors-scala-config.conf \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ {{- end }} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index b1c48924..ff505a84 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -14,9 +14,9 @@ commonLabels: # repository: sunbirded.azurecr.io/data-pipeline # tag: "release-5.2.0_RC1_2c615f8_12" # docker pull sunbirded.azurecr.io/sunbird-datapipeline:release-4.9.0_RC4_1 -registry: surabhi1510 -repository: flink-python-3.11 -tag: 1.0.0 +registry: sanketikahub +repository: flink-connectors +tag: 1.17.2-scala_2.12-java11 imagePullSecrets: [] ## Databases @@ -300,7 +300,8 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom -flink_jobs: [] +flink_jobs: + commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/Chart.yaml b/command-service/helm-charts/spark-connector-cron/Chart.yaml index a7a290f9..7c047bec 100644 --- a/command-service/helm-charts/spark-connector-cron/Chart.yaml +++ b/command-service/helm-charts/spark-connector-cron/Chart.yaml @@ -7,5 +7,5 @@ dependencies: description: A production-ready Helm chart base template maintainers: - name: NimbusHub.in -name: spark-submit-cron +name: spark-connector-cron version: 0.1.0 diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7d46e64c..9698dcc6 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -37,18 +37,16 @@ def execute(self, command_payload: CommandPayload, action: Action): return result def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): - result = None - namespaces = set() - for job_type, config in self.connector_job_config.items(): - namespaces.add(config["namespace"]) - for namespace in namespaces: - self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata, namespace) + result = None + self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) - return result + return result - def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata, namespace): + def _stop_connector_jobs(self, is_masterdata, namespace): + print(f"Uninstalling jobs for {namespace}..") managed_releases = [] + base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] connector_jar_config = self.config.find("connector_job") masterdata_jar_config = self.config.find("masterdata_job") for connector_type in connector_jar_config: @@ -67,14 +65,14 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata, nam jobs = helm_ls_result.stdout.decode() for job in jobs.splitlines()[1:]: release_name = job.split()[0] - if release_name in managed_releases: + if base_helm_chart in job: print("Uninstalling job {0}".format(release_name)) helm_uninstall_cmd = [ "helm", "uninstall", release_name, "--namespace", - self.connector_job_ns, + namespace, ] helm_uninstall_result = subprocess.run( helm_uninstall_cmd, @@ -116,7 +114,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): release_name = connector_instance.connector_id runtime = connector_instance.connector_runtime namespace = self.connector_job_config["flink"]["namespace"] - + job_name = release_name.replace(".", "-") helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( @@ -126,10 +124,10 @@ def _perform_flink_install(self, dataset_id, connector_instance): if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() print(jobs) - deployment_exists = any(release_name in line for line in jobs.splitlines()[1:]) + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: - restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-taskmanager --namespace {namespace}".format( - namespace=namespace, release_name=release_name + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, job_name=job_name ) print("Restart command: ", restart_cmd) # Run the helm command @@ -140,7 +138,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): shell=True, ) if helm_install_result.returncode == 0: - print(f"Job {release_name} re-deployment succeeded...") + print(f"Job {job_name} restart succeeded...") else: err = True return ActionResponse( @@ -155,50 +153,56 @@ def _perform_flink_install(self, dataset_id, connector_instance): return result else: - connector_source = json.loads(connector_instance.connector_source) - flink_jobs = { - "kafka-connector": { + if self._get_live_instances(runtime="flink", connector_instance=connector_instance): + connector_source = json.loads(connector_instance.connector_source) + flink_jobs = dict() + flink_jobs[release_name] = { "enabled": "true", - "job_classname": connector_source.get('main_class') + "source": connector_source.get("source"), + "main_program": connector_source.get("main_program") } - } - set_json_value = json.dumps(flink_jobs) - print("Kafka connector: ", set_json_value) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set-json", - f"flink_jobs={set_json_value}" - ] - - print(" ".join(helm_install_cmd)) + + set_json_value = json.dumps(flink_jobs) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + job_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set-json", + f"flink_jobs={set_json_value.replace(" ", "")}" + ] + + print(" ".join(helm_install_cmd)) - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} deployment succeeded...") - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", - ) - print( - f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + + print(helm_install_result) + + if helm_install_result.returncode == 0: + print(f"Job {job_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error installing job {job_name}: {helm_install_result.stderr.decode()}" + ) - if err is None: - result = ActionResponse(status="OK", status_code=200) + if err is None: + result = ActionResponse(status="OK", status_code=200) - return result + return result + else: + self._stop_connector_jobs(is_masterdata=False, namespace="flink") else: print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") return ActionResponse( @@ -221,113 +225,48 @@ def _perform_spark_install(self, dataset_id, connector_instance): "Yearly": "0 0 1 1 *" # Runs at midnight on January 1st each year } - # Define namespace namespace = self.connector_job_config["spark"]["namespace"] - - # Check if the job already exists - helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - helm_ls_result = subprocess.run(helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode() - job_exists = any(release_name in line for line in jobs.splitlines()[1:]) - - if job_exists: - if self._is_dataset_retired(dataset_id): - print("Dataset is retired. Uninstalling existing cron job.") - self._stop_connector_jobs(dataset_id, [], False, namespace) - else: - print("Updating existing job") - - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set", - "technology={}".format(connector_instance.technology), - "--set", - "instance_id={}".format(release_name), - "--set", - "connector_source={}".format(connector_source["source"]), - "--set", - "main_class={}".format(connector_source["main_class"]), - "--set", - "main_file={}".format(connector_source["main_program"]), - "--set", - "cronSchedule={}".format(schedule_configs[schedule]) - ] - - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} update succeeded...") - result = ActionResponse(status="OK", status_code=200) - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", - ) - print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") - else: - print("Installing new job") - - helm_install_cmd = [ - "helm", - "install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set", - "technology={}".format(connector_instance.technology), - "--set", - "instance_id={}".format(release_name), - "--set", - "connector_source={}".format(connector_source["source"]), - "--set", - "main_class={}".format(connector_source["main_class"]), - "--set", - "main_file={}".format(connector_source["main_program"]), - "--set", - "cronSchedule={}".format(schedule_configs[schedule]) - ] - - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} installation succeeded...") - result = ActionResponse(status="OK", status_code=200) - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", - ) - print(f"Error installing job {release_name}: {helm_install_result.stderr.decode()}") - + + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} update succeeded...") + result = ActionResponse(status="OK", status_code=200) else: - print(f"Error listing Spark jobs: {helm_ls_result.stderr.decode()}") + err = True result = ActionResponse( status="ERROR", status_code=500, - error_message="SPARK_HELM_LIST_EXCEPTION", + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) - + print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") + if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -348,7 +287,6 @@ def _get_connector_details(self, dataset_id): active_connectors.append(from_dict( data_class=ConnectorInstance, data=record )) - # connector_versions[connector_instance.id] = record['version'] return active_connectors @@ -366,37 +304,65 @@ def _get_masterdata_details(self, dataset_id): return is_masterdata - def _is_dataset_retired(self, dataset_id): - is_retired = False + ## TODO: check for connector_id as well + def _get_live_instances(self, runtime, connector_instance): + has_live_instances = False rows = self.db_service.execute_select_all( - f""" - SELECT status - FROM datasets - WHERE status='{DatasetStatusType.Retired.name}' AND dataset_id = '{dataset_id}' - """ - ) - for row in rows: - if row['status'] == DatasetStatusType.Retired.name: - print("Status: ",row['status']) - is_retired = True - break - - return is_retired - - def _get_live_instances(self, dataset_id, runtime, connector_instance): - active_connectors = [] - records = self.db_service.execute_select_all( f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id JOIN datasets d ON ci.dataset_id = d.id - WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}'; + WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}' AND ci.connector_id = '{connector_instance.connector_id}'; """ ) - - for record in records: - active_connectors.append(record['connector_instance_id']) + if len(rows) > 0: + has_live_instances = True - return active_connectors + return has_live_instances + + # def _perform_install(self, release): + # err = None + # result = None + # release_name = release["release_name"] + # helm_install_cmd = [ + # "helm", + # "upgrade", + # "--install", + # release_name, + # self.connector_job_chart_dir, + # "--namespace", + # self.connector_job_ns, + # "--create-namespace", + # "--set", + # "file.path={}".format(release["jar"]), + # "--set", + # "class.name={}".format(release["class"]), + # "--set", + # "job.name={}".format(release_name), + # "--set", + # "args={}".format(",".join(release["args"])), + # "--set", + # "schedule={}".format(release["schedule"]), + # ] + # helm_install_result = subprocess.run( + # helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + # ) + # if helm_install_result.returncode == 0: + # print(f"Job {release_name} deployment succeeded...") + # else: + # err = True + # result = ActionResponse( + # status="ERROR", + # status_code=500, + # error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + # ) + # print( + # f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + # ) + + # if err is None: + # result = ActionResponse(status="OK", status_code=200) + + # return result From 512c4359e509eb2de7107be236bfb67b152c3bbc Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 23 Aug 2024 13:27:42 +0530 Subject: [PATCH 084/311] #OBS-I167 : if dataset is empty return with error --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 63d751df..5ae41826 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -74,6 +74,9 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); + if(!dataset) { + return; + } const api_version = _.get(dataset, "api_version") let datasetConfigs: any = {} const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) From 6e350df904cadd614a9a8c506271b665f0628f5d Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 23 Aug 2024 13:40:32 +0530 Subject: [PATCH 085/311] #OBS-I143: inswert query fix --- command-service/src/command/connector_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index ae185174..1400668c 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -362,7 +362,7 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): datetime.now(), datetime.now(), - registry_meta.id, + registry_meta.id + "-" + registry_meta.version, registry_meta.name, registry_meta.type, registry_meta.category, From 98a7e3cdaf27ff28c953d03fc0efff7b193be42d Mon Sep 17 00:00:00 2001 From: Aniket Sakinala Date: Fri, 23 Aug 2024 18:35:25 +0530 Subject: [PATCH 086/311] Issue #OBS-I144 fix: icon data as string; check default version --- .../src/command/connector_registry.py | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index ae185174..0ebde996 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -1,9 +1,11 @@ import json import os +import base64 import tarfile import uuid import zipfile from datetime import datetime +from pathlib import Path import requests from fastapi import status @@ -184,6 +186,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: } connector_id = obj["id"].replace(" ", "-") + self.update_connector_registry(connector_id, self.metadata['metadata']['version']) registry_meta = ConnectorRegsitryv2(connector_id, obj['name'], 'source', @@ -194,7 +197,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: self.metadata['metadata']['runtime'], self.metadata['metadata']['licence'], self.metadata['metadata']['owner'], - obj['icon'], + self.load_file_bytes(obj["icon"]), 'Live', rel_path, json.dumps(source), @@ -225,7 +228,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: connector_id = ( self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") ) - + self.update_connector_registry(connector_id, self.metadata['metadata']['version']) source = { "source": connector_source, "main_class": self.metadata["metadata"]["main_class"], @@ -243,7 +246,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: self.metadata['metadata']['runtime'], self.metadata['metadata']['licence'], self.metadata['metadata']['owner'], - self.metadata['metadata']['icon'], + self.load_file_bytes(self.metadata['metadata']["icon"]), 'Live', rel_path, json.dumps(source), @@ -380,6 +383,43 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): datetime.now(), ) return query, params + + def load_file_bytes(self, rel_path: str) -> bytes | None: + file_path = Path(self.extraction_path) + for item in file_path.glob("*/{}".format(rel_path)): + try: + with open(item, 'rb') as file: + file_content = file.read() + encoded = base64.b64encode(file_content).decode("ascii") + except IsADirectoryError: + print( + f"Connector Registry | No value for icon URL given at metadata: {rel_path}" + ) + return None + except FileNotFoundError: + print( + f"Connector Registry | No file present at indicated relative path: {rel_path}" + ) + return None + except (ValueError or TypeError) as e: + print( + f"Connector Registry | File content not byte like: {e}" + ) + return None + return encoded + + def update_connector_registry(self, _id, ver): + try: + result = self.db_service.execute_upsert( + f"UPDATE connector_registry SET status = 'Retired', updated_date = now() WHERE connector_id = %s AND status = 'Live' AND version != %s", (_id, ver) + ) + print( + f"Connector Registry | Updated {result} existing rows with connector_id: {_id} and version: {ver}" + ) + except Exception as e: + print( + f"Connector Registry | An error occurred during the execution of Query: {e}" + ) class ExtractionUtil: def extract_gz(tar_path, extract_path): From 19a975b401f64b8a26316526f8449a2684be364d Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Mon, 26 Aug 2024 12:33:13 +0530 Subject: [PATCH 087/311] #OBS-143: fix: dataset publish fixes --- .../flink-connector/templates/deployment.yaml | 3 ++ .../src/command/connector_command.py | 44 ++++++++++--------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 6f4f98b7..3da5d6ce 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -173,6 +173,9 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: /opt/flink + env: + - name: CONNECTOR_ID + value: {{ index $jobData "connector_id" }} command: - /bin/bash - -c diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 9698dcc6..7b2d2b13 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -45,16 +45,17 @@ def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): def _stop_connector_jobs(self, is_masterdata, namespace): print(f"Uninstalling jobs for {namespace}..") - managed_releases = [] base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - connector_jar_config = self.config.find("connector_job") - masterdata_jar_config = self.config.find("masterdata_job") - for connector_type in connector_jar_config: - for release in connector_jar_config[connector_type]: - managed_releases.append(release["release_name"]) - if is_masterdata: - for release in masterdata_jar_config: - managed_releases.append(release["release_name"]) + + # managed_releases = [] + # connector_jar_config = self.config.find("connector_job") + # masterdata_jar_config = self.config.find("masterdata_job") + # for connector_type in connector_jar_config: + # for release in connector_jar_config[connector_type]: + # managed_releases.append(release["release_name"]) + # if is_masterdata: + # for release in masterdata_jar_config: + # managed_releases.append(release["release_name"]) helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( @@ -123,7 +124,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() - print(jobs) + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( @@ -156,8 +157,9 @@ def _perform_flink_install(self, dataset_id, connector_instance): if self._get_live_instances(runtime="flink", connector_instance=connector_instance): connector_source = json.loads(connector_instance.connector_source) flink_jobs = dict() - flink_jobs[release_name] = { + flink_jobs[job_name] = { "enabled": "true", + "connector_id": connector_instance.connector_id, "source": connector_source.get("source"), "main_program": connector_source.get("main_program") } @@ -176,7 +178,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): f"flink_jobs={set_json_value.replace(" ", "")}" ] - print(" ".join(helm_install_cmd)) + print("flink connector installation: ", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -185,7 +187,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): print(helm_install_result) if helm_install_result.returncode == 0: - print(f"Job {job_name} deployment succeeded...") + print(f"Job '{job_name}' deployment succeeded...") else: err = True result = ActionResponse( @@ -194,7 +196,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", ) print( - f"Error installing job {job_name}: {helm_install_result.stderr.decode()}" + f"Error installing job '{job_name}': {helm_install_result.stderr.decode()}" ) if err is None: @@ -250,13 +252,13 @@ def _perform_spark_install(self, dataset_id, connector_instance): "cronSchedule={}".format(schedule_configs[schedule]) ] - print(" ".join(helm_install_cmd)) + print("spark connector installation:", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) if helm_install_result.returncode == 0: - print(f"Job {release_name} update succeeded...") + print(f"Job '{release_name}' update succeeded...") result = ActionResponse(status="OK", status_code=200) else: err = True @@ -265,7 +267,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): status_code=500, error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) - print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") + print(f"Error updating job '{release_name}': {helm_install_result.stderr.decode()}") if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -307,15 +309,15 @@ def _get_masterdata_details(self, dataset_id): ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False - rows = self.db_service.execute_select_all( - f""" + query = f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id JOIN datasets d ON ci.dataset_id = d.id - WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}' AND ci.connector_id = '{connector_instance.connector_id}'; + WHERE cr.runtime = %s AND ci.status = %s AND ci.connector_id = %s; """ - ) + params = (runtime, DatasetStatusType.Live.name, connector_instance.connector_id) + rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: has_live_instances = True From f98b1aa52f2cf00f8c62dad4869ce0f2ff7ee9ad Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 26 Aug 2024 13:21:41 +0530 Subject: [PATCH 088/311] #OBS-I167 : fix: removed duplicate code. --- api-service/src/services/DatasetService.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 66a7edd6..9ec2012d 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -204,17 +204,9 @@ class DatasetService { draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { - const v1connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); - const modifiedV1Connectors = _.map(v1connectors, (config) => { - return { - id: _.get(config, "id"), - connector_id: _.get(config, "connector_type"), - connector_config: _.get(config, "connector_config"), - version: "v1" - } - }) + const v1connectors = await getV1Connectors(draftDataset.dataset_id); const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); - draftDataset["connectors_config"] = _.concat(modifiedV1Connectors, v2connectors) + draftDataset["connectors_config"] = _.concat(v1connectors, v2connectors) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } From b5e73069cc7fffd829a641ff8758ee814ae6fbad Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Tue, 27 Aug 2024 18:49:09 +0530 Subject: [PATCH 089/311] #OBS-I181 - Updated the event structure --- .../DataIngestion/DataIngestionController.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 49e1ebf6..415849fe 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -29,7 +29,7 @@ const dataIn = async (req: Request, res: Response) => { try { const requestBody = req.body; const datasetId = req.params.datasetId.trim(); - + const isValidSchema = schemaValidation(requestBody, validationSchema) if (!isValidSchema?.isValid) { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) @@ -68,18 +68,23 @@ const addMetadataToEvents = (datasetId: string, payload: any) => { const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source }; if (Array.isArray(validData)) { const payloadRef = validData.map((event: any) => { - event = _.set(event, "obsrv_meta", obsrvMeta); - event = _.set(event, "dataset", datasetId); - event = _.set(event, "msgid", mid); - return event + const payload = { + event, + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + } + return payload; }) return payloadRef; } else { - _.set(validData, "msgid", mid); - _.set(validData, "obsrv_meta", obsrvMeta); - _.set(validData, "dataset", datasetId); - return validData + return ({ + "event": validData, + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + }); } } From 77f20f4fbefe164deeba1bb36b513659b1c26c5b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 30 Aug 2024 19:34:41 +0530 Subject: [PATCH 090/311] #OBS-I164: added jwt token vwerification and access control to api's --- api-service/src/middlewares/jwtTokenVerify.ts | 156 ++++++++++++++++++ api-service/src/routes/Router.ts | 49 +++--- 2 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 api-service/src/middlewares/jwtTokenVerify.ts diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts new file mode 100644 index 00000000..09d6da62 --- /dev/null +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -0,0 +1,156 @@ +import { Request, Response, NextFunction } from "express"; +import jwt from "jsonwebtoken"; +import fs from "fs"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import _ from "lodash"; + +enum roles { + Admin = "admin", + DatasetManager = "dataset_manager", + Viewer = "viewer", + DatasetCreator = "dataset_creator", +} + +enum permissions { + DatasetCreate = "api.datasets.create", + DatasetUpdate = "api.datasets.update", + DatasetRead = "api.datasets.read", + DatasetList = "api.datasets.list", + DataIngest = "api.data.in", + DataOut = "api.data.out", + DataExhaust = "api.data.exhaust", + QueryTemplateCreate = "api.query.template.create", + QueryTemplateRead = "api.query.template.read", + QueryTemplateDelete = "api.query.template.delete", + QueryTemplateList = "api.query.template.list", + QueryTemplateUpdate = "api.query.template.update", + QueryTemplate = "api.query.template.query", + SchemaValidator = "api.schema.validator", + GenerateSignedUrl = "api.files.generate-url", + DatasetStatusTransition = "api.datasets.status-transition", + DatasetHealth = "api.dataset.health", + DatasetReset = "api.dataset.reset", + DatasetSchemaGenerator = "api.datasets.dataschema", + DatasetExport = "api.datasets.export", + DatasetCopy = "api.datasets.copy", + ConnectorList = "api.connectors.list", + ConnectorRead = "api.connectors.read", + DatasetImport = "api.datasets.import", + SQLQuery = "api.obsrv.data.sql-query" +} + +interface AccessControl { + [key: string]: string[]; +} + +const accessControl: AccessControl = { + [roles.Admin]: [ + permissions.DatasetRead, + permissions.DatasetList, + permissions.DatasetHealth, + permissions.DatasetReset, + permissions.QueryTemplateList, + permissions.ConnectorRead, + permissions.ConnectorList, + permissions.SQLQuery, + ], + [roles.DatasetManager]: [ + permissions.DatasetUpdate, + permissions.DatasetRead, + permissions.DatasetList, + permissions.DatasetHealth, + permissions.DatasetReset, + permissions.DatasetCopy, + permissions.QueryTemplateRead, + permissions.QueryTemplateUpdate, + permissions.QueryTemplateDelete, + permissions.QueryTemplateList, + permissions.SchemaValidator, + permissions.ConnectorRead, + permissions.ConnectorList, + permissions.SQLQuery, + ], + [roles.Viewer]: [ + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetRead, + permissions.DatasetList, + permissions.DatasetHealth, + permissions.DatasetReset, + permissions.QueryTemplateRead, + permissions.QueryTemplateList, + permissions.ConnectorRead, + permissions.ConnectorList, + permissions.SQLQuery, + ], + [roles.DatasetCreator]: [ + permissions.DatasetImport, + permissions.DataIngest, + permissions.DataOut, + permissions.DatasetCreate, + permissions.DatasetRead, + permissions.DatasetUpdate, + permissions.DatasetList, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SQLQuery, + permissions.DatasetStatusTransition, + permissions.DatasetSchemaGenerator, + ], +}; + +export default { + name: "jwt:tokenAuthorization", + handler: () => (req: Request, res: Response, next: NextFunction) => { + try { + const public_key = fs.readFileSync("public_key.pem", "utf8"); + const authHeader = req.headers.authorization; + const token = authHeader && authHeader.split(" ")[1]; + if (!token) { + ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "No token provided", + }, + req, + res + ); + } + jwt.verify(token as string, public_key, (err, decoded) => { + if (err) { + ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "Token verification failed", + }, + req, + res + ); + } + if (decoded && typeof decoded == "object") { + const action = (req as any).id; + const hasAccess = decoded.roles.some((role: string) => accessControl[role] && accessControl[role].includes(action)); + if (!hasAccess) { + ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "Access denied for the user", + }, + req, + res + ); + } + next(); + } + }); + } catch (error) { + next(error); + } + }, +}; diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1faaafb0..fa91cac4 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,33 +29,34 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import {OperationType, telemetryAuditStart} from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; +import jwtTokenVerify from "../middlewares/jwtTokenVerify"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), jwtTokenVerify.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), jwtTokenVerify.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), jwtTokenVerify.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), jwtTokenVerify.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), jwtTokenVerify.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), jwtTokenVerify.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), jwtTokenVerify.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), jwtTokenVerify.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), jwtTokenVerify.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), jwtTokenVerify.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), jwtTokenVerify.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetImport); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file From 6aa8a8f11d0b258768b8bce8b28d53ffbcdbedad Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 30 Aug 2024 19:37:20 +0530 Subject: [PATCH 091/311] #OBS-I164: added jwt token vwerification and access control to api's --- api-service/src/routes/Router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index fa91cac4..0c65be4a 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -59,4 +59,4 @@ router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read") router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), sqlQuery); \ No newline at end of file From 9d31325666f292162f37518e65d2af81a8a263e8 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Sep 2024 17:51:13 +0530 Subject: [PATCH 092/311] #OBS-I164: modified the access roles and permissions --- api-service/src/middlewares/jwtTokenVerify.ts | 79 ++++++++++++------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts index 09d6da62..51601b8f 100644 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -9,6 +9,7 @@ enum roles { DatasetManager = "dataset_manager", Viewer = "viewer", DatasetCreator = "dataset_creator", + Ingestor = "ingestor" } enum permissions { @@ -44,61 +45,79 @@ interface AccessControl { } const accessControl: AccessControl = { - [roles.Admin]: [ - permissions.DatasetRead, + [roles.Ingestor]: [ + permissions.DataIngest, + ], + [roles.Viewer]: [ permissions.DatasetList, - permissions.DatasetHealth, - permissions.DatasetReset, - permissions.QueryTemplateList, + permissions.DatasetRead, + permissions.DatasetExport, permissions.ConnectorRead, - permissions.ConnectorList, permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, ], - [roles.DatasetManager]: [ - permissions.DatasetUpdate, - permissions.DatasetRead, + [roles.DatasetCreator]: [ permissions.DatasetList, - permissions.DatasetHealth, - permissions.DatasetReset, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, permissions.DatasetCopy, + permissions.QueryTemplateCreate, permissions.QueryTemplateRead, - permissions.QueryTemplateUpdate, permissions.QueryTemplateDelete, - permissions.QueryTemplateList, + permissions.GenerateSignedUrl, permissions.SchemaValidator, + permissions.DatasetSchemaGenerator + ], + [roles.DatasetManager]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, permissions.ConnectorRead, - permissions.ConnectorList, permissions.SQLQuery, - ], - [roles.Viewer]: [ permissions.DataOut, permissions.DataExhaust, - permissions.DatasetRead, - permissions.DatasetList, - permissions.DatasetHealth, - permissions.DatasetReset, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, permissions.QueryTemplateRead, - permissions.QueryTemplateList, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition + ], + [roles.Admin]: [ + permissions.DatasetCreate, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, permissions.ConnectorRead, - permissions.ConnectorList, permissions.SQLQuery, - ], - [roles.DatasetCreator]: [ - permissions.DatasetImport, - permissions.DataIngest, permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, permissions.DatasetCreate, - permissions.DatasetRead, permissions.DatasetUpdate, - permissions.DatasetList, permissions.DatasetCopy, permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, permissions.GenerateSignedUrl, - permissions.SQLQuery, - permissions.DatasetStatusTransition, + permissions.SchemaValidator, permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition ], }; From 859a0cf823c6c2ac34f341e581c3220ee3dd8bbe Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Sep 2024 18:27:03 +0530 Subject: [PATCH 093/311] #OBS-I164: reading public key from env file --- api-service/src/middlewares/jwtTokenVerify.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts index 51601b8f..3834e0ef 100644 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -1,8 +1,7 @@ import { Request, Response, NextFunction } from "express"; import jwt from "jsonwebtoken"; -import fs from "fs"; import { ResponseHandler } from "../helpers/ResponseHandler"; -import _ from "lodash"; +import { config } from "../configs/Config"; enum roles { Admin = "admin", @@ -125,7 +124,7 @@ export default { name: "jwt:tokenAuthorization", handler: () => (req: Request, res: Response, next: NextFunction) => { try { - const public_key = fs.readFileSync("public_key.pem", "utf8"); + const public_key = config.jwt_public_key; const authHeader = req.headers.authorization; const token = authHeader && authHeader.split(" ")[1]; if (!token) { From eebc2f93a4b931b825fa5f3ce78eefb75be7b53a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 10:34:07 +0530 Subject: [PATCH 094/311] #OBS-I186 : fix: dataset metrics api --- .../QueryWrapper/NativeQueryWrapper.ts | 21 ++++++++++++++ .../nativeQueryValidationSchema.json | 12 ++++++++ api-service/src/routes/Router.ts | 28 ++++++++++--------- 3 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts create mode 100644 api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json diff --git a/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts new file mode 100644 index 00000000..c6ea4573 --- /dev/null +++ b/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts @@ -0,0 +1,21 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./nativeQueryValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; + +const nativeQuery = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const query = _.get(req, ["body", "query", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); +} + +export default nativeQuery; \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json b/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json new file mode 100644 index 00000000..ae17e302 --- /dev/null +++ b/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "query": { + "type": "object", + "nullable": true + } + }, + "required": [ + "query" + ] +} \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1faaafb0..14dc40ff 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,35 +27,37 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; -import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; +import nativeQuery from "../controllers/QueryWrapper/NativeQueryWrapper"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), dataIn); router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.updateDataset, operationType: OperationType.UPDATE }), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.readDataset, operationType: OperationType.GET }), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.listDatasets, operationType: OperationType.LIST }), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.datasetExhaust, operationType: OperationType.GET }), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); router.post("/template/list", setDataToRequestObject("api.query.template.list"), listQueryTemplates); router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.createTransformation, operationType: OperationType.CREATE }), DatasetStatusTansition); router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.copyDataset, operationType: OperationType.CREATE }), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.listConnectors, operationType: OperationType.GET }), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.readConnectors, operationType: OperationType.GET }), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), nativeQuery) \ No newline at end of file From 38bcad9a515236bcb87eb6df3eff32f1feb50d48 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 10:35:29 +0530 Subject: [PATCH 095/311] #OBS-I185 : fix: removed duplicate code. --- .../src/connections/druidConnection.ts | 2 +- .../DataIngestion/DataIngestionController.ts | 23 +++----- .../controllers/DataOut/DataOutController.ts | 59 ++++++++----------- api-service/src/services/DatasetService.ts | 4 +- api-service/src/services/TableGenerator.ts | 18 +++--- .../src/command/connector_command.py | 2 +- command-service/src/command/db_command.py | 4 +- command-service/src/command/druid_command.py | 2 +- 8 files changed, 46 insertions(+), 68 deletions(-) diff --git a/api-service/src/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts index 7ca7b2dc..a25e40d8 100644 --- a/api-service/src/connections/druidConnection.ts +++ b/api-service/src/connections/druidConnection.ts @@ -3,7 +3,7 @@ import * as _ from "lodash"; import { config } from "../configs/Config"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); -const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +export const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; const sqlQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.sql_query_path}`; export const executeNativeQuery = async (payload: any) => { diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 415849fe..69785067 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -26,8 +26,8 @@ const apiId = "api.data.in"; const errorCode = "DATASET_UPDATE_FAILURE" const dataIn = async (req: Request, res: Response) => { - try { - const requestBody = req.body; + + const requestBody = req.body; const datasetId = req.params.datasetId.trim(); const isValidSchema = schemaValidation(requestBody, validationSchema) @@ -35,29 +35,20 @@ const dataIn = async (req: Request, res: Response) => { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); } - const dataset = await datasetService.getDataset(datasetId, ["id"], true) + const dataset = await datasetService.getDataset(datasetId, ["id", "entry_topic", "api_version", "dataset_config"], true) if (!dataset) { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); } - const entryTopic = _.get(dataset, "dataValues.dataset_config.entry_topic") + const { entry_topic, dataset_config, api_version } = dataset + const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic if (!entryTopic) { logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); } - await send(addMetadataToEvents(datasetId, requestBody), _.get(dataset, "dataValues.dataset_config.entry_topic")) + await send(addMetadataToEvents(datasetId, requestBody), entryTopic) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); - } - catch (err: any) { - const code = _.get(err, "code") || errorCode - logger.error({ ...err, apiId, code }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "DATA_INGESTION_FAILED", message: "Failed to ingest data" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } + } const addMetadataToEvents = (datasetId: string, payload: any) => { diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts index a61bf545..dd6f93b6 100644 --- a/api-service/src/controllers/DataOut/DataOutController.ts +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -12,44 +12,33 @@ const dataOut = async (req: Request, res: Response) => { const datasetId = req.params?.datasetId; const requestBody = req.body; const msgid = _.get(req, "body.params.msgid"); - try { - const isValidSchema = schemaValidation(requestBody, validationSchema); - if (!isValidSchema?.isValid) { - logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); - } - const isValidQuery: any = await validateQuery(req.body, datasetId); - const query = _.get(req, "body.query", "") - - if (isValidQuery === true && _.isObject(query)) { - const result = await executeNativeQuery(query); - logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } + const isValidSchema = schemaValidation(requestBody, validationSchema); + if (!isValidSchema?.isValid) { + logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); + } + const isValidQuery: any = await validateQuery(req.body, datasetId); + const query = _.get(req, "body.query", "") - if (isValidQuery === true && _.isString(query)) { - const result = await executeSqlQuery({ query }) - logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } + if (isValidQuery === true && _.isObject(query)) { + const result = await executeNativeQuery(query); + logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); + } - else { - logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) - return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); - } + if (isValidQuery === true && _.isString(query)) { + const result = await executeSqlQuery({ query }) + logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); } - catch (err: any) { - logger.error({ ...err, apiId, code: err?.code || "INTERNAL_SERVER_ERROR" }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "INTERNAL_SERVER_ERROR", message: "Unable to process the query." } - } - ResponseHandler.errorResponse(errorMessage, req, res); + + else { + logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) + return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); } } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 9ec2012d..2e800212 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -315,7 +315,7 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.id, "PUBLISH_DATASET"); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); } @@ -354,7 +354,7 @@ class DatasetService { return { id: _.join([datasource, type], "_"), datasource: draftDataset.dataset_id, - dataset_id: draftDataset.dataset_id, + dataset_id: draftDataset.id, datasource_ref: datasource, type } diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index eb3065ae..e22714f6 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -143,16 +143,14 @@ class TableGenerator extends BaseTableGenerator { } private getDruidFlattenSpec = (allFields: Record) => { - return _.union( - _.map(allFields, (field) => { - return { - type: "path", - expr: field.expr, - name: field.name - } - }), - rawIngestionSpecDefaults.flattenSpec - ) + const allfields = _.map(allFields, (field) => { + return { + type: "path", + expr: field.expr, + name: field.name + } + }); + return _.uniqBy([...allfields, ...rawIngestionSpecDefaults.flattenSpec], "name") } getHudiIngestionSpecForCreate = (dataset: Record, allFields: Record[], datasourceRef: string) => { diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7b2d2b13..b18c1c33 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -175,7 +175,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set-json", - f"flink_jobs={set_json_value.replace(" ", "")}" + f"""flink_jobs={set_json_value.replace(" ", "")}""" ] print("flink connector installation: ", " ".join(helm_install_cmd)) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 4cb62e2c..89db3392 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -176,7 +176,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): draft_datasource.datasource, dataset_id, draft_datasource.datasource_ref, - json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), json.dumps(draft_datasource.archival_policy).replace("'", "''"), @@ -191,7 +191,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): json.dumps(draft_datasource.metadata).replace("'", "''"), draft_datasource.datasource, - json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), json.dumps(draft_datasource.archival_policy).replace("'", "''"), diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index 901b18ad..f43222b4 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -35,7 +35,7 @@ def _submit_ingestion_task(self, dataset_id): f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." ) for record in datasources_records: - if record["dataset_type"] == "dataset" and record["type"] == "druid": + if record["dataset_type"] == "event" and record["type"] == "druid": print(f"Submitting ingestion task for datasource ...") ingestion_spec = json.dumps(record["ingestion_spec"]) response = self.http_service.post( From 2d0afbec43da18f9df122986c6f26bda64c8c503 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 10:52:24 +0530 Subject: [PATCH 096/311] #OBS-I164: modified public key variable in config --- api-service/src/configs/Config.ts | 5 +++-- api-service/src/middlewares/jwtTokenVerify.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 38fe624c..6a28dc37 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -4,7 +4,7 @@ const env = process.env.system_env || "local" export const config = { "env": env, - "api_port": process.env.api_port || 3000, + "api_port": process.env.api_port || 3005, "body_parser_limit": process.env.body_parser_limit || "100mb", "version": "1.0", "query_api": { @@ -112,5 +112,6 @@ export const config = { "dialect": process.env.dialet || "postgres", "url": process.env.grafana_url || "http://localhost:8000", "access_token": process.env.grafana_token || "" - } + }, + "user_token_public_key": process.env.user_token_public_key || "", } diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts index 3834e0ef..99823112 100644 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -124,7 +124,7 @@ export default { name: "jwt:tokenAuthorization", handler: () => (req: Request, res: Response, next: NextFunction) => { try { - const public_key = config.jwt_public_key; + const public_key = config.user_token_public_key; const authHeader = req.headers.authorization; const token = authHeader && authHeader.split(" ")[1]; if (!token) { From 06525852836e9f2e43b03725fd933d263f0070d4 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Tue, 3 Sep 2024 13:17:19 +0530 Subject: [PATCH 097/311] flink connector helm chart updates --- .../flink-connector/templates/deployment.yaml | 60 ++++++----------- .../helm-charts/flink-connector/values.yaml | 66 ++----------------- 2 files changed, 27 insertions(+), 99 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 3da5d6ce..41405e4b 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -49,7 +49,7 @@ spec: containers: - name: {{ $jobName }}-taskmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: {{ .Values.taskmanager.flink_work_dir }} args: ["taskmanager"] env: @@ -69,33 +69,14 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - # - name: flink-config-volume - # mountPath: /data/flink/conf/connectors-scala-config.conf - # subPath: connectors-scala-config.conf - # - name: flink-config-volume - # mountPath: /data/flink/conf/connectors-python-config.conf - # subPath: connectors-python-config.yaml - # - name: flink-config-volume - # mountPath: /opt/flink/conf/flink-conf.yaml - # subPath: flink-conf.yaml - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties volumes: - # - name: flink-config-volume - # configMap: - # name: flink-connector-conf - # items: - # - key: connectors-scala-config.conf - # path: connectors-scala-config.conf - # - key: connectors-python-config.yaml - # path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config items: - # - key: flink-conf - # path: flink-conf.yaml - key: log4j_console_properties path: log4j-console.properties {{ $component := "jobmanager" }} @@ -143,7 +124,7 @@ spec: containers: - name: {{ $jobName }}-jobmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: /opt/flink # command: ["/bin/sh", "-c"] args: ["jobmanager"] @@ -157,34 +138,38 @@ spec: metrics.reporter.prom.host: {{ $jobName }}-jobmanager metrics.reporter.prom.port: 9250 volumeMounts: - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf subPath: connectors-scala-config.conf - - name: flink-config-volume - mountPath: /data/flink/conf/connectors-python-config.conf - subPath: connectors-python-config.conf + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml - name: data - mountPath: /flink/connectors + mountPath: /data/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - name: {{ $jobName }}-job-submit image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: /opt/flink env: - name: CONNECTOR_ID value: {{ index $jobData "connector_id" }} + - name: CONFIG_PATH + value: /data/flink/conf/ + - name: CONFIG_FILE + value: connectors-python-config.yaml command: - /bin/bash - -c - | - /usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py; + /usr/bin/python3 /data/connectors-init/connector.py; sleep 30s; - /opt/flink/bin/flink run -m \ + /opt/flink/bin/flink run -d -m \ {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ - /flink/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ + /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ --config.file.path /data/flink/conf/connectors-scala-config.conf \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ @@ -198,7 +183,7 @@ spec: {{- if eq .Values.checkpoint_store_type "gcs" }} "-Dgoogle.cloud.auth.service.account.enable=true" \ {{- end }} - ; + ; sleep 10s; echo "Job submitted"; tail -f /dev/null; ports: {{- range .Values.service.ports }} - name: {{ .name }} @@ -208,23 +193,21 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf subPath: connectors-scala-config.conf - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-python-config.yaml subPath: connectors-python-config.yaml - name: data - mountPath: /flink/connectors + mountPath: /data/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - - volumes: - - name: flink-config-volume + - name: connector-config-volume configMap: - name: flink-connector-conf + name: connectors-config items: - key: connectors-scala-config.conf path: connectors-scala-config.conf @@ -248,7 +231,6 @@ spec: resources: requests: storage: 2Gi - storageClassName: "standard" {{- end }} {{- end}} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index ff505a84..dd08dafa 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -18,6 +18,7 @@ registry: sanketikahub repository: flink-connectors tag: 1.17.2-scala_2.12-java11 imagePullSecrets: [] +imagePullPolicy: IfNotPresent ## Databases global: @@ -205,65 +206,6 @@ log4j_console_properties: | logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline logger.netty.level = OFF -baseconfig: | - kafka { - broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - producer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - consumer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - zookeeper = "{{ .Values.global.zookeeper.host }}:{{ .Values.global.zookeeper.port }}" - producer { - max-request-size = 1572864 - batch.size = 98304 - linger.ms = 10 - compression = "snappy" - } - output.system.event.topic = ${job.env}".system.events" - - output.failed.topic = ${job.env}".failed" - } - job { - env = "{{ .Values.global.env }}" - enable.distributed.checkpointing = {{ .Values.checkpointing.enabled }} - statebackend { - base.url = "{{ .Values.checkpointing.statebackend }}" - } - } - - task { - parallelism = 1 - consumer.parallelism = 1 - checkpointing.interval = 10000 - checkpointing.pause.between.seconds = 10000 - restart-strategy.attempts = 3 - restart-strategy.delay = 30000 # in milli-seconds - } - - redis.connection.timeout = 100 - redis { - host = "{{ .Values.global.redis_dedup.host }}" - port = {{ .Values.global.redis_dedup.port }} - } - - redis-meta { - host = "{{ .Values.global.redis_denorm.host }}" - port = {{ .Values.global.redis_denorm.port }} - } - - postgres { - host = "{{ .Values.global.postgresql.host }}" - port = "{{ .Values.global.postgresql.port }}" - maxConnections = "{{ .Values.postgres.max_connections }}" - sslMode = "{{ .Values.postgres.sslmode }}" - user = "{{ .Values.global.postgresql.obsrv.user }}" - password = "{{ .Values.global.postgresql.obsrv.password }}" - database = "{{ .Values.global.postgresql.obsrv.name }}" - } - - lms-cassandra { - host = "{{ .Values.global.cassandra.host }}" - port = "{{ .Values.global.cassandra.port }}" - } - flink_resources: taskmanager: resources: @@ -301,7 +243,11 @@ serviceMonitor: port: prom flink_jobs: - + "kafka-connector-1-0-0": + "enabled": "false" + "connector_id": "kafka-connector-1.0.0" + "source": "kafka-connector-1.0.0" + "main_program": "kafka-connector-1.0.0.jar" commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file From 9e108d8297e6a9437a388b5ad5550db817358dd4 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Tue, 3 Sep 2024 15:29:14 +0530 Subject: [PATCH 098/311] flink connector helm chart updates --- .../flink-connector/templates/deployment.yaml | 1 + .../src/command/connector_command.py | 48 +++++++++---------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 41405e4b..74fc9cca 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -171,6 +171,7 @@ spec: {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ --config.file.path /data/flink/conf/connectors-scala-config.conf \ + --metadata.id {{ index $jobData "connector_id" }} \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ {{- end }} diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7b2d2b13..aabb9db4 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -37,16 +37,16 @@ def execute(self, command_payload: CommandPayload, action: Action): return result def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): - result = None + result = None self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) - return result + return result def _stop_connector_jobs(self, is_masterdata, namespace): print(f"Uninstalling jobs for {namespace}..") base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - + # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -101,14 +101,14 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): f"Connector {connector.connector_id} is not supported for deployment" ) break - + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") # for release in masterdata_jar_config: # result = self._perform_install(release) return result - + def _perform_flink_install(self, dataset_id, connector_instance): err = None result = None @@ -117,14 +117,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace = self.connector_job_config["flink"]["namespace"] job_name = release_name.replace(".", "-") helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - + helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - + if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() - + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( @@ -148,14 +148,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): error_message="FLINK_HELM_LIST_EXCEPTION", ) print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") - + if err is None: result = ActionResponse(status="OK", status_code=200) return result else: if self._get_live_instances(runtime="flink", connector_instance=connector_instance): - connector_source = json.loads(connector_instance.connector_source) + connector_source = connector_instance.connector_source flink_jobs = dict() flink_jobs[job_name] = { "enabled": "true", @@ -163,7 +163,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): "source": connector_source.get("source"), "main_program": connector_source.get("main_program") } - + set_json_value = json.dumps(flink_jobs) helm_install_cmd = [ "helm", @@ -175,17 +175,17 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set-json", - f"flink_jobs={set_json_value.replace(" ", "")}" + f"""flink_jobs={set_json_value.replace(" ", "")}""" ] - + print("flink connector installation: ", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - + print(helm_install_result) - + if helm_install_result.returncode == 0: print(f"Job '{job_name}' deployment succeeded...") else: @@ -212,14 +212,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): status_code=500, error_message="FLINK_HELM_LIST_EXCEPTION", ) - + def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None release_name = connector_instance.id - connector_source = json.loads(connector_instance.connector_source) + connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] - + schedule_configs = { "Hourly": "0 * * * *", # Runs at the start of every hour "Weekly": "0 0 * * 0", # Runs at midnight every Sunday @@ -228,7 +228,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): } namespace = self.connector_job_config["spark"]["namespace"] - + helm_install_cmd = [ "helm", "upgrade", @@ -268,7 +268,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) print(f"Error updating job '{release_name}': {helm_install_result.stderr.decode()}") - + if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -305,11 +305,11 @@ def _get_masterdata_details(self, dataset_id): is_masterdata = True return is_masterdata - + ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False - query = f""" + query = f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id @@ -320,9 +320,9 @@ def _get_live_instances(self, runtime, connector_instance): rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: has_live_instances = True - + return has_live_instances - + # def _perform_install(self, release): # err = None # result = None From f669b5126da2986d495e723fb6e9b004a9c283d9 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 3 Sep 2024 17:47:04 +0530 Subject: [PATCH 099/311] fix: dataset publish fixes --- .../templates/cronjob.yaml | 1 + .../src/command/connector_registry.py | 71 ++++++++++++++++++- command-service/src/model/db_models.py | 2 +- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index 07b70fb2..7d35b32b 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -8,6 +8,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: + concurrencyPolicy: Forbid schedule: {{ .Values.cronSchedule }} jobTemplate: spec: diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index a91f651d..862b55e2 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -8,6 +8,7 @@ from pathlib import Path import requests +import subprocess from fastapi import status from config import Config @@ -175,7 +176,9 @@ def load_ui_metadata(self, extract_out_path): def process_metadata(self, rel_path, connector_source) -> RegistryResponse: result = [] tenant = self.metadata.get("metadata", {}).get("tenant", "") - + + self.copy_connector_to_runtime(self.metadata['metadata']['runtime'], connector_source) + if tenant == "multiple": connector_objects = self.metadata["connectors"] for obj in connector_objects: @@ -210,6 +213,10 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + if not success: return RegistryResponse( status="failure", @@ -259,6 +266,10 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + if not success: return RegistryResponse( status="failure", @@ -272,6 +283,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_200_OK, ) + def execute_query(self, query, params) -> bool: try: result = self.db_service.execute_upsert(sql=query, params=params) @@ -420,6 +432,63 @@ def update_connector_registry(self, _id, ver): print( f"Connector Registry | An error occurred during the execution of Query: {e}" ) + + def copy_connector_to_runtime(self, runtime: str, connector_source: str): + if runtime == "spark": + return self.copy_connector_to_spark(connector_source) + + def copy_connector_to_spark(self, connector_source: str): + print(f"Connector Registry | copying {connector_source} to spark") + ## get name of the spark pod using kubectl + spark_pod_cmd = [ + "kubectl", + "get", + "pods", + "-n", + "spark", + "-l", + "app.kubernetes.io/name=spark,app.kubernetes.io/component=master", + "-o", + "jsonpath='{.items[0].metadata.name}'", + ] + + spark_pod_result = subprocess.run(spark_pod_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + print(f"Connector Registry | spark_pod_result: {spark_pod_result}") + + if spark_pod_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to get the spark pod", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + spark_pod = spark_pod_result.stdout.decode("utf-8").replace("'", "") + print(f"Connector Registry | spark_pod: {spark_pod}") + + ## copy the connector to the spark pod under /data/connectors/{source} + source_path = os.path.join(self.extraction_path, connector_source) + copy_cmd = [ + "kubectl", + "cp", + f"{source_path}", + f"spark/{spark_pod}:/data/connectors/{connector_source}", + ] + + copy_result = subprocess.run(copy_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | copy_result: {copy_result}") + if copy_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to copy the connector to spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + return RegistryResponse( + status="success", + message="connector copied to spark successfully", + statusCode=status.HTTP_200_OK, + ) class ExtractionUtil: def extract_gz(tar_path, extract_path): diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index f77f8314..6586c52d 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -123,5 +123,5 @@ class ConnectorInstance: connector_id: str operations_config: dict connector_runtime: str - connector_source: str + connector_source: dict technology: str \ No newline at end of file From f5f366727429fe1da06583b8c0ec83639ebee9e1 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 17:53:16 +0530 Subject: [PATCH 100/311] #OBS-I164: modified public key variable in config --- api-service/src/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 6a28dc37..7adafd0f 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -4,7 +4,7 @@ const env = process.env.system_env || "local" export const config = { "env": env, - "api_port": process.env.api_port || 3005, + "api_port": process.env.api_port || 3000, "body_parser_limit": process.env.body_parser_limit || "100mb", "version": "1.0", "query_api": { From 808ce80d3c7971cfb60c8afe1c752915964c5d92 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 18:08:27 +0530 Subject: [PATCH 101/311] #OBS-I186 : added dataset mertric api controller and route and minor change in dataset transition api --- .../DatasetMetricsController.ts | 38 +++++ .../DatasetMetricsValidationSchema.json} | 0 .../DatasetStatusTransition.ts | 53 +++---- .../ReadyToPublishSchema.json | 130 ++++++++++++++---- .../QueryWrapper/NativeQueryWrapper.ts | 21 --- api-service/src/routes/Router.ts | 4 +- 6 files changed, 171 insertions(+), 75 deletions(-) create mode 100644 api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts rename api-service/src/controllers/{QueryWrapper/nativeQueryValidationSchema.json => DatasetMetrics/DatasetMetricsValidationSchema.json} (100%) delete mode 100644 api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts new file mode 100644 index 00000000..0e0d2193 --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DatasetMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, '/prom')) return config.query_api.prometheus.url + _.replace(url, '/prom', '') +} + +const datasetMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + let { query } = req.body || {}; + let endpoint = query.url; + if (_.startsWith(endpoint, '/prom')) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, 'data'); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json b/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json similarity index 100% rename from api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json rename to api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 5c86d519..eb07934c 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -24,7 +24,7 @@ const allowedTransitions: Record = { } const liveDatasetActions = ["Retire", "Archive", "Purge"] -const validateRequest = (req: Request, datasetId: any) => { +const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) @@ -32,7 +32,7 @@ const validateRequest = (req: Request, datasetId: any) => { } const validateDataset = (dataset: any, datasetId: any, action: string) => { - + if (_.isEmpty(dataset)) { throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${datasetId}`, "NOT_FOUND", 404) } @@ -41,7 +41,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "BAD_REQUEST", 400) } - if(!_.includes(allowedTransitions[action], dataset.status)) { + if (!_.includes(allowedTransitions[action], dataset.status)) { throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "CONFLICT", 409) } @@ -54,10 +54,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) validateDataset(dataset, dataset_id, status); - switch(status) { + switch (status) { case "Delete": await deleteDataset(dataset); break; @@ -74,7 +74,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await archiveDataset(dataset); break; case "Purge": - await purgeDataset(dataset); + await purgeDataset(dataset); break; } @@ -94,11 +94,12 @@ const readyForPublish = async (dataset: Record) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) - defaultConfigs = _.omit(defaultConfigs, ["router_config","dedupe_config"]) + defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); if (draftDataset?.type === 'master') { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } - _.mergeWith(draftDataset,defaultConfigs,draftDataset, (objValue, srcValue) => { + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { return objValue; } @@ -128,7 +129,7 @@ const readyForPublish = async (dataset: Record) => { const publishDataset = async (dataset: Record) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - + await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) @@ -138,9 +139,9 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) // 1. Check if there are denorm fields and dependent master datasets are published const denormConfig = _.get(draftDataset, "denorm_config") - if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { const datasetIds = _.map(denormConfig.denorm_fields, "dataset_id") - if(_.includes(datasetIds, draftDataset.id)) { + if (_.includes(datasetIds, draftDataset.id)) { throw { code: "SELF_REFERENCING_MASTER_DATA", message: `The denorm master dataset is self-referencing itself`, @@ -148,26 +149,26 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) statusCode: 409 } } - const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ id: datasetIds, type: "master" }, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) - const datasetStatus : Record = { + const datasetStatus: Record = { dataset_id: denormField.dataset_id, exists: (md) ? true : false, - isLive: (md) ? md.status === "Live" : false, + isLive: (md) ? md.status === "Live" : false, status: (md) ? md.status : false } - if(!_.isEmpty(md)){ - if(md.api_version === "v2") - datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); - else - datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + if (!_.isEmpty(md)) { + if (md.api_version === "v2") + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.cache_config.redis_db }); + else + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.redis_db }); } return datasetStatus; }) - const invalidMasters = _.filter(masterDatasetsStatus, {isLive: false}) - if(_.size(invalidMasters) > 0) { + const invalidMasters = _.filter(masterDatasetsStatus, { isLive: false }) + if (_.size(invalidMasters) > 0) { const invalidIds = _.map(invalidMasters, "dataset_id") throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", @@ -193,7 +194,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() - if(_.isEmpty(results)) { + if (_.isEmpty(results)) { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", message: `Unable to fetch the redis db index for the master data`, @@ -222,14 +223,14 @@ const canRetireIfMasterDataset = async (dataset: Record) => { const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["dataset_config", "dataset_id"]) || [] const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) - const extractDenormFields = _.map(allDatasets, function(depDataset) { - return {dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id")} + const extractDenormFields = _.map(allDatasets, function (depDataset) { + return { dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id") } }) - const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) + const deps = _.filter(extractDenormFields, function (depDS) { return _.includes(depDS.denorm_datasets, dataset.id) }) if (_.size(deps) > 0) { const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` - throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, "denorm_datasets")})) + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function (o) { return _.omit(o, "denorm_datasets") })) } } } diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index bbb67fdb..eff208cb 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -14,11 +14,17 @@ }, "status": { "type": "string", - "enum": ["Draft"] + "enum": [ + "Draft" + ] }, "type": { "type": "string", - "enum": ["event", "transaction", "master"] + "enum": [ + "event", + "transaction", + "master" + ] }, "name": { "type": "string", @@ -36,10 +42,16 @@ }, "mode": { "type": "string", - "enum": ["Strict", "IgnoreNewFields"] + "enum": [ + "Strict", + "IgnoreNewFields" + ] } }, - "required": ["validate", "mode"] + "required": [ + "validate", + "mode" + ] }, "extraction_config": { "type": "object", @@ -78,9 +90,15 @@ "dedup_key": { "minLength": 1 } - } + }, + "required": [ + "dedup_key", + "dedup_period" + ] }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], + "required": [ + "drop_duplicates" + ], "additionalProperties": false } }, @@ -133,9 +151,15 @@ "dedup_key": { "minLength": 1 } - } + }, + "required": [ + "dedup_key", + "dedup_period" + ] }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], + "required": [ + "drop_duplicates" + ], "additionalProperties": false }, "data_schema": { @@ -160,7 +184,10 @@ "type": "boolean" } }, - "required": ["type", "properties"], + "required": [ + "type", + "properties" + ], "additionalProperties": false }, "denorm_config": { @@ -200,17 +227,27 @@ }, "oneOf": [ { - "required": ["dataset_id", "denorm_out_field", "denorm_key"] + "required": [ + "dataset_id", + "denorm_out_field", + "denorm_key" + ] }, { - "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + "required": [ + "dataset_id", + "denorm_out_field", + "jsonata_expr" + ] } ], "additionalProperties": false } } }, - "required": ["denorm_fields"], + "required": [ + "denorm_fields" + ], "additionalProperties": false }, "router_config": { @@ -221,7 +258,9 @@ "minLength": 1 } }, - "required": ["topic"], + "required": [ + "topic" + ], "additionalProperties": false }, "dataset_config": { @@ -253,9 +292,13 @@ "default": false } }, - "required": ["olap_store_enabled", "lakehouse_enabled", "cache_enabled"], + "required": [ + "olap_store_enabled", + "lakehouse_enabled", + "cache_enabled" + ], "additionalProperties": false - }, + }, "keys_config": { "type": "object", "properties": { @@ -273,7 +316,10 @@ "minLength": 1 } }, - "required": ["data_key", "timestamp_key"], + "required": [ + "data_key", + "timestamp_key" + ], "additionalProperties": false }, "cache_config": { @@ -290,11 +336,18 @@ "type": "integer" } }, - "required": ["redis_db_host", "redis_db_port"], + "required": [ + "redis_db_host", + "redis_db_port" + ], "additionalProperties": false } }, - "required": ["indexing_config", "keys_config", "cache_config"], + "required": [ + "indexing_config", + "keys_config", + "cache_config" + ], "additionalProperties": false }, "tags": { @@ -337,7 +390,10 @@ "type": "string" } }, - "required": ["type", "expr"], + "required": [ + "type", + "expr" + ], "additionalProperties": false }, "datatype": { @@ -345,19 +401,34 @@ }, "category": { "type": "string", - "enum": ["pii", "transform", "derived"] + "enum": [ + "pii", + "transform", + "derived" + ] } }, - "required": ["type", "expr", "category"], + "required": [ + "type", + "expr", + "category" + ], "additionalProperties": false }, "mode": { "type": "string", - "enum": ["Strict", "Lenient"] + "enum": [ + "Strict", + "Lenient" + ] } }, "additionalProperties": false, - "required": ["field_key", "transformation_function", "mode"] + "required": [ + "field_key", + "transformation_function", + "mode" + ] } }, "connectors_config": { @@ -392,7 +463,12 @@ } }, "additionalProperties": false, - "required": ["id", "connector_id", "connector_config", "version"] + "required": [ + "id", + "connector_id", + "connector_config", + "version" + ] } } }, @@ -413,7 +489,9 @@ "minLength": 1 } }, - "required": ["data_key"] + "required": [ + "data_key" + ] } } } @@ -437,4 +515,4 @@ "transformations_config", "connectors_config" ] -} +} \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts deleted file mode 100644 index c6ea4573..00000000 --- a/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Request, Response } from "express"; -import _ from "lodash"; -import { executeNativeQuery } from "../../connections/druidConnection"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import vaidationSchema from "./nativeQueryValidationSchema.json" -import { schemaValidation } from "../../services/ValidationService"; -import logger from "../../logger"; -import { obsrvError } from "../../types/ObsrvError"; - -const nativeQuery = async (req: Request, res: Response) => { - const isValidSchema = schemaValidation(req.body, vaidationSchema); - if (!isValidSchema?.isValid) { - logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) - throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) - } - const query = _.get(req, ["body", "query", "query"]); - const response = await executeNativeQuery(query); - ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); -} - -export default nativeQuery; \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 14dc40ff..d40ed232 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,7 +29,7 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import nativeQuery from "../controllers/QueryWrapper/NativeQueryWrapper"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; export const router = express.Router(); @@ -60,4 +60,4 @@ router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), o //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); -router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), nativeQuery) \ No newline at end of file +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) \ No newline at end of file From 9a08837d5730c11b82b055f5ce8c6ec84a39051c Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 3 Sep 2024 18:35:20 +0530 Subject: [PATCH 102/311] install pip requirments if applicable --- .../src/command/connector_registry.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 862b55e2..ceb2cbba 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -436,6 +436,7 @@ def update_connector_registry(self, _id, ver): def copy_connector_to_runtime(self, runtime: str, connector_source: str): if runtime == "spark": return self.copy_connector_to_spark(connector_source) + def copy_connector_to_spark(self, connector_source: str): print(f"Connector Registry | copying {connector_source} to spark") @@ -483,7 +484,29 @@ def copy_connector_to_spark(self, connector_source: str): message="failed to copy the connector to spark", statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - + + if self.metadata['metadata']['technology'] == "python": + pip_install_cmd = [ + "kubectl", + "exec", + f"pod/{spark_pod}", + "-n", + "spark", + "--", + "bash", + "-c", + f"pip install -r /data/connectors/{connector_source}/requirements.txt", + ] + + pip_install_result = subprocess.run(pip_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | pip_install_result: {pip_install_result}") + if pip_install_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to install the requirements on spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + return RegistryResponse( status="success", message="connector copied to spark successfully", From 588245bd9d447636a3e392ce0098452277e1b3d9 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 18:42:53 +0530 Subject: [PATCH 103/311] #OBS-I186 : removed export statement --- api-service/src/connections/druidConnection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts index a25e40d8..7ca7b2dc 100644 --- a/api-service/src/connections/druidConnection.ts +++ b/api-service/src/connections/druidConnection.ts @@ -3,7 +3,7 @@ import * as _ from "lodash"; import { config } from "../configs/Config"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); -export const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; const sqlQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.sql_query_path}`; export const executeNativeQuery = async (payload: any) => { From 4f5e155f29fd42ffc0eb31ca703922c31f9d06a0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 18:45:36 +0530 Subject: [PATCH 104/311] #OBS-I164: added config for option rbac verification --- api-service/src/configs/Config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 7adafd0f..81702432 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -114,4 +114,5 @@ export const config = { "access_token": process.env.grafana_token || "" }, "user_token_public_key": process.env.user_token_public_key || "", + "is_RBAC_enabled": false } From 9d16185a53b4824779431057f7e503acde9894fb Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 18:46:27 +0530 Subject: [PATCH 105/311] #OBS-I164: changed the middleware to rbac_middleware --- .../src/middlewares/RBAC_middleware.ts | 178 ++++++++++++++++++ api-service/src/middlewares/jwtTokenVerify.ts | 174 ----------------- api-service/src/routes/Router.ts | 52 ++--- 3 files changed, 204 insertions(+), 200 deletions(-) create mode 100644 api-service/src/middlewares/RBAC_middleware.ts delete mode 100644 api-service/src/middlewares/jwtTokenVerify.ts diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts new file mode 100644 index 00000000..b84b2edb --- /dev/null +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -0,0 +1,178 @@ +import { Request, Response, NextFunction } from "express"; +import jwt from "jsonwebtoken"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import { config } from "../configs/Config"; + +enum roles { + Admin = "admin", + DatasetManager = "dataset_manager", + Viewer = "viewer", + DatasetCreator = "dataset_creator", + Ingestor = "ingestor", +} + +enum permissions { + DatasetCreate = "api.datasets.create", + DatasetUpdate = "api.datasets.update", + DatasetRead = "api.datasets.read", + DatasetList = "api.datasets.list", + DataIngest = "api.data.in", + DataOut = "api.data.out", + DataExhaust = "api.data.exhaust", + QueryTemplateCreate = "api.query.template.create", + QueryTemplateRead = "api.query.template.read", + QueryTemplateDelete = "api.query.template.delete", + QueryTemplateList = "api.query.template.list", + QueryTemplateUpdate = "api.query.template.update", + QueryTemplate = "api.query.template.query", + SchemaValidator = "api.schema.validator", + GenerateSignedUrl = "api.files.generate-url", + DatasetStatusTransition = "api.datasets.status-transition", + DatasetHealth = "api.dataset.health", + DatasetReset = "api.dataset.reset", + DatasetSchemaGenerator = "api.datasets.dataschema", + DatasetExport = "api.datasets.export", + DatasetCopy = "api.datasets.copy", + ConnectorList = "api.connectors.list", + ConnectorRead = "api.connectors.read", + DatasetImport = "api.datasets.import", + SQLQuery = "api.obsrv.data.sql-query", +} + +interface AccessControl { + [key: string]: string[]; +} + +const accessControl: AccessControl = { + [roles.Ingestor]: [permissions.DataIngest], + [roles.Viewer]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + ], + [roles.DatasetCreator]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + ], + [roles.DatasetManager]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition, + ], + [roles.Admin]: [ + permissions.DatasetCreate, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition, + ], +}; + +export default { + name: "rbac:middleware", + handler: () => (req: Request, res: Response, next: NextFunction) => { + try { + if (config.is_RBAC_enabled === true) { + const public_key = config.user_token_public_key; + const token = req.get("x-user-token"); + if (!token) { + return ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "No token provided", + }, + req, + res + ); + } + jwt.verify(token as string, public_key, (err, decoded) => { + if (err) { + return ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "Token verification failed", + }, + req, + res + ); + } + if (decoded && typeof decoded == "object") { + const action = (req as any).id; + const hasAccess = decoded.roles.some( + (role: string) => + accessControl[role] && accessControl[role].includes(action) + ); + if (!hasAccess) { + return ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "Access denied for the user", + }, + req, + res + ); + } + next(); + } + }); + } else { + next(); + } + } catch (error) { + next(error); + } + }, +}; diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts deleted file mode 100644 index 99823112..00000000 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import jwt from "jsonwebtoken"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { config } from "../configs/Config"; - -enum roles { - Admin = "admin", - DatasetManager = "dataset_manager", - Viewer = "viewer", - DatasetCreator = "dataset_creator", - Ingestor = "ingestor" -} - -enum permissions { - DatasetCreate = "api.datasets.create", - DatasetUpdate = "api.datasets.update", - DatasetRead = "api.datasets.read", - DatasetList = "api.datasets.list", - DataIngest = "api.data.in", - DataOut = "api.data.out", - DataExhaust = "api.data.exhaust", - QueryTemplateCreate = "api.query.template.create", - QueryTemplateRead = "api.query.template.read", - QueryTemplateDelete = "api.query.template.delete", - QueryTemplateList = "api.query.template.list", - QueryTemplateUpdate = "api.query.template.update", - QueryTemplate = "api.query.template.query", - SchemaValidator = "api.schema.validator", - GenerateSignedUrl = "api.files.generate-url", - DatasetStatusTransition = "api.datasets.status-transition", - DatasetHealth = "api.dataset.health", - DatasetReset = "api.dataset.reset", - DatasetSchemaGenerator = "api.datasets.dataschema", - DatasetExport = "api.datasets.export", - DatasetCopy = "api.datasets.copy", - ConnectorList = "api.connectors.list", - ConnectorRead = "api.connectors.read", - DatasetImport = "api.datasets.import", - SQLQuery = "api.obsrv.data.sql-query" -} - -interface AccessControl { - [key: string]: string[]; -} - -const accessControl: AccessControl = { - [roles.Ingestor]: [ - permissions.DataIngest, - ], - [roles.Viewer]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - ], - [roles.DatasetCreator]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator - ], - [roles.DatasetManager]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition - ], - [roles.Admin]: [ - permissions.DatasetCreate, - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition - ], -}; - -export default { - name: "jwt:tokenAuthorization", - handler: () => (req: Request, res: Response, next: NextFunction) => { - try { - const public_key = config.user_token_public_key; - const authHeader = req.headers.authorization; - const token = authHeader && authHeader.split(" ")[1]; - if (!token) { - ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "No token provided", - }, - req, - res - ); - } - jwt.verify(token as string, public_key, (err, decoded) => { - if (err) { - ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "Token verification failed", - }, - req, - res - ); - } - if (decoded && typeof decoded == "object") { - const action = (req as any).id; - const hasAccess = decoded.roles.some((role: string) => accessControl[role] && accessControl[role].includes(action)); - if (!hasAccess) { - ResponseHandler.errorResponse( - { - statusCode: 401, - errCode: "Unauthorized access", - message: "Access denied for the user", - }, - req, - res - ); - } - next(); - } - }); - } catch (error) { - next(error); - } - }, -}; diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 0c65be4a..d2e53b3b 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,34 +29,34 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import {OperationType, telemetryAuditStart} from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import jwtTokenVerify from "../middlewares/jwtTokenVerify"; +import rbacVerify from "../middlewares/RBAC_middleware"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(),DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), jwtTokenVerify.handler(), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), jwtTokenVerify.handler(), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), jwtTokenVerify.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), jwtTokenVerify.handler(), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), jwtTokenVerify.handler(), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), jwtTokenVerify.handler(), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), jwtTokenVerify.handler(), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), jwtTokenVerify.handler(), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), jwtTokenVerify.handler(), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), jwtTokenVerify.handler(), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), jwtTokenVerify.handler(), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetImport); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), rbacVerify.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), rbacVerify.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), rbacVerify.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), rbacVerify.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), rbacVerify.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), rbacVerify.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), rbacVerify.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), rbacVerify.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), rbacVerify.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), rbacVerify.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), rbacVerify.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), sqlQuery); \ No newline at end of file From 5d8563371372cbb3b045f44c7f187551916a9c13 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 19:19:50 +0530 Subject: [PATCH 106/311] #OBS-I186 : Logic moved to separate function --- .../controllers/DatasetRead/DatasetRead.ts | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 5ae41826..e34a996b 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -36,19 +36,7 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset?.connectors_config.map((connector: any) => { - let connector_config = _.get(connector, "connector_config") - const authMechanism = _.get(connector_config, ["authenticationMechanism"]) - if (authMechanism && authMechanism.encrypted) { - connector_config = { - ...connector_config, - authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues))} - } - return { - ...connector, - connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) - } - }); + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } @@ -74,7 +62,7 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); - if(!dataset) { + if (!dataset) { return; } const api_version = _.get(dataset, "api_version") @@ -104,4 +92,23 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { + return connectorsConfig.map((connector: any) => { + let connector_config = _.get(connector, "connector_config"); + const authMechanism = _.get(connector_config, ["authenticationMechanism"]); + + if (authMechanism && authMechanism.encrypted) { + connector_config = { + ...connector_config, + authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues)) + }; + } + + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) + }; + }); +} + export default datasetRead; \ No newline at end of file From ac09f850a5d11ded6b601fa7136a165082c60764 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 19:22:18 +0530 Subject: [PATCH 107/311] #OBS-I164: changed import name --- api-service/src/routes/Router.ts | 52 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index d2e53b3b..bf66e34d 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,34 +29,34 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import {OperationType, telemetryAuditStart} from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import rbacVerify from "../middlewares/RBAC_middleware"; +import checkRBAC from "../middlewares/RBAC_middleware"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(),DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), rbacVerify.handler(), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), rbacVerify.handler(), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), rbacVerify.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), rbacVerify.handler(), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), rbacVerify.handler(), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), rbacVerify.handler(), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), rbacVerify.handler(), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), rbacVerify.handler(), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), rbacVerify.handler(), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), rbacVerify.handler(), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), rbacVerify.handler(), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetImport); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), checkRBAC.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), checkRBAC.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), checkRBAC.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), checkRBAC.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); \ No newline at end of file From 40049e93f4390b41a8752b973dd3d872aeb41190 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 19:55:57 +0530 Subject: [PATCH 108/311] #OBS-I186 : Logic moved to separate function --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index e34a996b..d2a362d3 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -36,7 +36,7 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } From 7fbcf742e214c3bd64d9cd92fefb838a1dbbf692 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 19:56:18 +0530 Subject: [PATCH 109/311] #OBS-I185 : fix: test case fixes --- .../DataIngestion/DataIngestionController.ts | 1 - .../controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 2 +- .../DataIngestTest/DataIngestionTest.spec.ts | 61 ++++++++++++------- .../DataOutTest/DataQueryTest.spec.ts | 8 +-- .../DatasetRead/DatasetRead.spec.ts | 29 +++++++-- 6 files changed, 70 insertions(+), 33 deletions(-) diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 69785067..46f829e9 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -23,7 +23,6 @@ const errorObject = { } } const apiId = "api.data.in"; -const errorCode = "DATASET_UPDATE_FAILURE" const dataIn = async (req: Request, res: Response) => { diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 5ae41826..025ef05f 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -78,7 +78,7 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise) => { let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); - if (draftDataset?.type === 'master') { + if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { diff --git a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts index 487cd77e..1ff1c9b4 100644 --- a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts @@ -36,15 +36,13 @@ describe("DATA INGEST API", () => { it("it should ingest data for individual event", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - }, - extraction_config: { - is_batch_event: false, - extraction_key: "events", - batch_id: "id" - } + dataset_config: { + entry_topic: "local.test.topic", + }, + extraction_config: { + is_batch_event: false, + extraction_key: "events", + batch_id: "id" } }) }) @@ -71,10 +69,8 @@ describe("DATA INGEST API", () => { it("it should ingest data successfully", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -97,13 +93,37 @@ describe("DATA INGEST API", () => { }) }); + it("it should ingest data successfully v2", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ + api_version: "v2", + entry_topic: "local.test.topic", + }) + }) + const connectionStub = sinon.stub(kafkaModule, "connect").returns(true); + const sendStub = sinon.stub(kafkaModule, "send").returns(resultResponse); + chai + .request(app) + .post(apiEndpoint) + .send(TestInputsForDataIngestion.SAMPLE_INPUT_1) + .end((err, res) => { + res.should.have.status(200); + res.body.should.be.a("object"); + res.body.should.have.property("result"); + res.body.id.should.be.eq("api.data.in"); + res.body.result.message.should.be.eq("Data ingested successfully") + connectionStub.restore() + sendStub.restore() + chai.spy.restore(Dataset, "findOne") + done() + }) + }); + it("Failed to connect kafka.", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -125,9 +145,7 @@ describe("DATA INGEST API", () => { it("Entry topic not found", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: {} - } + dataset_config: {} }) }) @@ -195,8 +213,7 @@ describe("DATA INGEST API", () => { res.body.should.be.a("object") res.body.id.should.be.eq("api.data.in"); res.body.params.status.should.be.eq("FAILED"); - res.body.error.code.should.be.eq("DATA_INGESTION_FAILED"); - res.body.error.message.should.be.eq("Failed to ingest data") + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 96d53e38..9f5cf5b3 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -89,8 +89,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); @@ -115,8 +115,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index cfd333eb..c3e766e7 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -31,9 +31,18 @@ describe("DATASET READ API", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "name": "sb-telemetry", "version": 1 }) }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=name,version") + .get("/v2/datasets/read/sb-telemetry?fields=name,version,connectors_config,transformations_config") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -42,7 +51,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: "sb-telemetry", version: 1 })) + result.should.be.eq(JSON.stringify({ "name": "sb-telemetry", "version": 1, "connectors_config": [], "transformations_config": [] })) done(); }); }); @@ -69,6 +78,15 @@ describe("DATASET READ API", () => { }); it("Dataset read success: Fetch live dataset when mode param not provided", (done) => { + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) @@ -83,7 +101,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.status.should.be.eq("Live") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA, connectors_config: TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2, transformations_config: TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA })) done(); }); }); @@ -98,6 +116,9 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetTransformations, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) @@ -267,7 +288,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":20}}}]) + return Promise.resolve([{ "dataset_id": "master_dataset", "dataset_config": { "cache_config": { "redis_db": 20 } } }]) }) chai .request(app) From 7e330d384828f445b6f671b1e935ebbf8852b4e0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 20:02:55 +0530 Subject: [PATCH 110/311] #OBS-I185 : fix: linting fix --- .../DatasetMetrics/DatasetMetricsController.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 0e0d2193..903d393c 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -10,7 +10,7 @@ import axios from "axios"; import { config } from "../../configs/Config"; const getBaseUrl = (url: string) => { - if (_.startsWith(url, '/prom')) return config.query_api.prometheus.url + _.replace(url, '/prom', '') + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") } const datasetMetrics = async (req: Request, res: Response) => { @@ -19,13 +19,13 @@ const datasetMetrics = async (req: Request, res: Response) => { logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) } - let { query } = req.body || {}; - let endpoint = query.url; - if (_.startsWith(endpoint, '/prom')) { + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { query.url = getBaseUrl(endpoint) const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) - const data = _.get(apiResponse, 'data'); + const data = _.get(apiResponse, "data"); return res.json(data); } else { From 7dff8607580e16d23e3736814fab259a9afa355a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 20:03:12 +0530 Subject: [PATCH 111/311] #OBS-I164: modified config and rbac middleware --- api-service/src/configs/Config.ts | 2 +- api-service/src/middlewares/RBAC_middleware.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 81702432..b26a6906 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -114,5 +114,5 @@ export const config = { "access_token": process.env.grafana_token || "" }, "user_token_public_key": process.env.user_token_public_key || "", - "is_RBAC_enabled": false + "is_RBAC_enabled": process.env.is_rbac_enabled || "false", } diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index b84b2edb..d36babf0 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -2,6 +2,7 @@ import { Request, Response, NextFunction } from "express"; import jwt from "jsonwebtoken"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; +import _ from "lodash"; enum roles { Admin = "admin", @@ -122,7 +123,9 @@ export default { name: "rbac:middleware", handler: () => (req: Request, res: Response, next: NextFunction) => { try { - if (config.is_RBAC_enabled === true) { + if (_.lowerCase(config.is_RBAC_enabled) === "false") { + next(); + } else { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { @@ -148,9 +151,9 @@ export default { res ); } - if (decoded && typeof decoded == "object") { + if (decoded && _.isObject(decoded)) { const action = (req as any).id; - const hasAccess = decoded.roles.some( + const hasAccess = decoded?.roles?.some( (role: string) => accessControl[role] && accessControl[role].includes(action) ); @@ -168,8 +171,6 @@ export default { next(); } }); - } else { - next(); } } catch (error) { next(error); From b83a8082f7b234e68667526d35d47a1a31c33d9e Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 20:15:28 +0530 Subject: [PATCH 112/311] #OBS-I164: added jsonwebtoken package --- api-service/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/package.json b/api-service/package.json index 1d49c029..acde61cc 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -33,6 +33,7 @@ "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", + "jsonwebtoken": "^9.0.1", "kafka-node": "^5.0.0", "kafkajs": "^2.2.4", "kafkajs-snappy": "^1.1.0", @@ -65,6 +66,7 @@ "@types/compression": "^1.7.2", "@types/express": "^4.17.14", "@types/http-errors": "^2.0.1", + "@types/jsonwebtoken": "^9.0.6", "@types/kafkajs": "^1.9.0", "@types/knex": "^0.16.1", "@types/lodash": "^4.14.190", From 95fe03b41a2e632b71c9614d7b9543f86f070b37 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Thu, 5 Sep 2024 15:13:20 +0530 Subject: [PATCH 113/311] V2 apis (#240) * #OBS-I115: Dataset list API refactoring * #0000: adding command api * #OBS-I116: Dataset CRUD APIs test and fixes * #OBS-I115: cmd api remove addn modules * #OBS-I116: Dataset status trasition to retire check for denorm fields * #OBS-I116: Dataset CRUD APIs test cases and fixes * #OBS-I115: Dataset Transition API refactoring * #OBS-I115: Dataset Transition API refactoring * #OBS-I116: Dataset update API Dedupe and denorm test cases fixes * #OBS-I115: Dataset Transition API refactoring and error handling refactoring * #OBS-I115: Dataset Transition API refactoring * #OBS-I115: Dataset Transition API refactoring * #OBS-I115: Remove unnecessary field fields_set * #OBS-I115: Dataset publish API - update the index of hudi spec properly for publish to handle schema evolution * #OBS-I116: Dataset create and status transition api code fix * #OBS-I116: Dataset Create api test case fixes * #OBS-I116: Dataset update extraction config api test case fixes * #OBS-I116: Dataset druid ingestion spec generation fix * #OBS-I116: express version upgraded * #OBS-I116: Dataset create api fixes * #OBS-I116: Dataset ingestion spec generation fix * #OBS-I126: updated swagger documentation * #OBS-I126: updated postman collection * #OBS-I126: updated postman collection * #OBS-I116: fix: entry topic column in datasets model * #OBS-I58 feat: Minio cloud store support - Added endpoint. as optional config to support the minio * #OBS-I126: added dataset read, list, update api's documentation and updated collection * #OBS-I116: feat: Dataschema api implementation v2 * #OBS-I116: feat: Schema validation fix * #OBS-I126: swagger doc updated * #OBS-I116: Dataset update api test cases * #OBS-I116: fix: linting fixes * #OBS-I116: lint fixes * #OBS-I116: Dataset status transition test cases * #OBS-I126 : added multiple requests example * #OBS-I126 : updated order * #OBS-I126 : updated server url * #OBS-I21: feat: dataset publish changes for connectors * #OBS-I1 updated Dataset Health API code * #OBS-I116: Dataset CRUD api fixes * #OBS-I116: fix: error codes fix * #OBS-I1 Refactored as per new changes * #OBS-I101: Update the publish API for the v2 APIs * #OBS-I101: fix db models * #OBS-I115: Remove the v1 unused API code and restructure the folders * #OBS-I1 Added DatasethealthService * #OBS-I1 Updated the imports and folders * #OBS-I2 updated dataset reset * #OBS-I116: fix: Command api and schema fixes * #OBS-I116: fix: Command api fix in db query * #OBS-I1 Added Notifications and alerts APIs * #OBS-I138: added decrypted response for the read api for connectors_config field and added defaults updated date and created date to list api * #OBS-I116: fix: feat: Dataset copy and export api implementation * #OBS-I2 Refactoring as per v2 APIs * #OBS-I2 typo fix * #OBS-I116: fix: fix: Dataset copy check for dataset fix * #OBS-I21: dataset publish changes fixes * #OBS-I116: fix: feat: Feedback fixes of removing set redis db * #OBS-I138: added cors to app * #OBS-I116: fix: feat: Dataset import api implementation * #OBS-I116: fix: fix: Dataset service fix * #OBS-I116: fix: repeated Validation method removal * #OBS-I116: fix: unused code * connector list * #OBS-I142: connector list api * #OBS-I116: fix: schema validation check for v1 exported dataset. * #OBS-I116: fix: Dataset overwrite after creation failure * #OBS-I116: fix: error handling * #OBS-I116: fix: error messages fix * #OBS-I116: fix: code fixes * #OBS-I142: formatted connector list api files * #OBS-I142: formatted connector list api files * #OBS-I142: formatted connector list file * #OBS-I138: required changes for dataset for master dataset migration from v1 to v2 * #OBS-I138: removed cors package * #OBS-I138: updated package json file * #OBS-I138: indentation fix * #OBS-I138: indentation fix * #OBS-I138: fixed indentations * #OBS-I142: updated postman collection * #OBS-I142: added connector list swagger documentation * #OBS-I142: updated postman collection * #OBS-I116: feat: Dataset Import and export api integration fixes * #OBS-I138: removed comment * #OBS-I138: throwing error if dataset is undefined * #OBS-I142: added test cases for connector list * #OBS-I138: merging dataset defaults to dataset draft record before saving * #OBS-I138: adding merged event to dataset while migrating live or draft dataset * #OBS-I142: added live_date field to defaultFields * #OBS-I145: Connector Read API * #OBS-I145: updated postman collection with connector read api * #OBS-I145: updated swagger documentation * #OBS-I145: updated the connector read api * merge changes * merge changes * Resolved merge changes * Resolved merge changes * #OBS-I145: updated the connector read api * #OBS-I138: removed datakey before merging defaults to dataset * #OBS-I145: updated postman collection and swagger documentation * #OBS-I145: added the test cases for connector read api * #OBS-I145: added a test case for connector read api * #OBS-I116: feat: Dataset import api fixes * #OBS-I116: feat: Test cases and linting fixes * #OBS-I116: feat: Dataset status transition test cases fix * #OBS-I1 updated the routes * #OBS-I116: feat: Dataset migratio method fix * #OBS-I108: feat: helm modifications for flink connectors * #OBS-I108: helm chart fixes * #OBS-I108: feat: Modify volume mounts * #OBS-I108: feat: Add PVC for JobManager * #OBS-I108: feat: change args for jobmanager command * #OBS-I141: added a new metric to sum the response time * #OBS-I141: modified the url variable and access dataset_id from params * #OBS-I141: added helper function to get dataset_id for error cases * #OBS-I108: feat: Use sidecar container to submit connector flink job * #OBS-I141: added telemetry for v2 api's * #OBS-I141: added a new metric to sum the response time * #OBS-I141: modified the url variable and access dataset_id from params * #OBS-I141: added helper function to get dataset_id for error cases * #OBS-I141: added telemetry for v2 api's * #OBS-I141: added telemetry for v2 api's * #OBS-I143: feat: dataset publish changes to deploy flink connectors * #OBS-I141: removed metric for sum of response time * #OBS-I141: removed usage of builtin kafka methods from telemetry file * #OBS-I146: feat: Retire fix * Issue #SBCOSS-12 fix: convert all SQL raw queries to prepared statements * Issue #SBCOSS-12 fix: tags is an array, so requires empty json for null case; dataset draft deletion requires deletion of transformation and source config drafts * #SBCOSS-23: feat: dataset publish changes for redeployment * #OBS-I167 : read api changes while reading connectors according to v2 structure * #OBS-I173: fix: Ready to publish schema fix to expect connector configs as object and string * #OBS-I174: fix: Dataset read api fix to expect both v1 and v2 connectors * #OBS-I146: fix: Test case fix for read api * #OBS-I146: fix: Test case fix for read api changes * #OBS-I146: fix: status transition test cases * #OBS-I146: fix: Test case script fix * #OBS-I146: fix: Type error fix * #OBS-I141: removed metric for sum of response time * #OBS-I146: fix: Dataset read api test cases fixes * #OBS-I146: fix: Hudi spec generation test cases * #OBS-I146: fix: Test case and linting fix * merge commit * #OBS-I173: fix: Dataset web console required fixes * #OBS-I173: fix: Dataset update changes to accept type changes * #OBS-I167 : dataset read api changes to read live dataset source configs * #OBS-I146: fix: linting fix * #OBS-I146: fix: linting fix * #OBS-I167 : Added string or dict as type to connector_config * #OBS-I143: dataset publish changes fixes * #OBS-I167 : if dataset is empty return with error * #OBS-I143: inswert query fix * Issue #OBS-I144 fix: icon data as string; check default version * #OBS-143: fix: dataset publish fixes * #OBS-I167 : fix: removed duplicate code. * #OBS-I181 - Updated the event structure * #OBS-I164: added jwt token vwerification and access control to api's * #OBS-I164: added jwt token vwerification and access control to api's * #OBS-I164: modified the access roles and permissions * #OBS-I164: reading public key from env file * #OBS-I186 : fix: dataset metrics api * #OBS-I185 : fix: removed duplicate code. * #OBS-I164: modified public key variable in config * flink connector helm chart updates * flink connector helm chart updates * fix: dataset publish fixes * #OBS-I164: modified public key variable in config * #OBS-I186 : added dataset mertric api controller and route and minor change in dataset transition api * install pip requirments if applicable * #OBS-I186 : removed export statement * #OBS-I164: added config for option rbac verification * #OBS-I164: changed the middleware to rbac_middleware * #OBS-I186 : Logic moved to separate function * #OBS-I164: changed import name * #OBS-I186 : Logic moved to separate function * #OBS-I185 : fix: test case fixes * #OBS-I185 : fix: linting fix * #OBS-I164: modified config and rbac middleware * #OBS-I164: added jsonwebtoken package * master url fix --------- Co-authored-by: Santhosh Vasabhaktula Co-authored-by: JeraldJF Co-authored-by: yashashk Co-authored-by: harishkumar gangula Co-authored-by: SurabhiAngadi Co-authored-by: Rakshitha-D Co-authored-by: Anand Parthasarathy Co-authored-by: Aniket Sakinala --- .github/workflows/pull_request.yaml | 1 - .gitignore | 18 +- api-service/.eslintignore | 2 +- api-service/.eslintrc | 3 +- api-service/package-lock.json | 24055 ---------------- api-service/package.json | 24 +- .../updated_v2_collection.json | 2701 ++ api-service/src/app.ts | 53 +- api-service/src/{v2 => }/configs/Config.ts | 33 +- .../src/{v2 => }/configs/ConnectionsConfig.ts | 0 .../src/configs/DatasetConfigDefault.ts | 60 + api-service/src/configs/IngestionConfig.ts | 79 + .../src/{v1 => }/configs/QueryRules.ts | 0 .../connections/commandServiceConnection.ts | 21 + .../connections/databaseConnection.ts | 11 +- .../{v2 => }/connections/druidConnection.ts | 4 +- .../src/connections/grafanaConnection.ts | 10 + .../{v2 => }/connections/kafkaConnection.ts | 16 +- api-service/src/controllers/Alerts/Alerts.ts | 145 + api-service/src/controllers/Alerts/Metric.ts | 86 + api-service/src/controllers/Alerts/Silence.ts | 117 + .../ConnectorsList/ConnectorsList.ts | 38 + .../ConnectorsListValidationSchema.json | 73 + .../ConnectorsRead/ConnectorsRead.ts | 22 + .../CreateTemplateController.ts | 0 .../CreateTemplateValidationSchema.json | 0 .../QueryTemplateValidator.ts | 0 .../DataExhaust/DataExhaustController.ts | 5 +- .../DataIngestion/DataIngestionController.ts | 50 +- .../DataIngestion/validationSchema.json | 0 .../controllers/DataOut/DataOutController.ts | 45 + .../DataOut/DataOutValidationSchema.json | 0 .../controllers/DataOut/QueryRules.ts | 0 .../controllers/DataOut/QueryValidator.ts | 0 .../controllers/DatasetCopy/DatasetCopy.ts | 53 + .../DatasetCopy/DatasetCopyHelper.ts | 17 + .../DatasetCopy/RequestValidationSchema.json | 57 + .../DatasetCreate/DatasetCreate.ts | 91 + .../DatasetCreateValidationSchema.json} | 290 +- .../DatasetExport/DatasetExport.ts | 38 + .../DatasetHealth/DatasetHealth.ts | 49 + .../DatasetHealthValidationSchema.json | 53 + .../DatasetImport/DatasetImport.ts | 66 + .../DatasetImport/DatasetImportHelper.ts | 154 + .../RequestValidationSchemaV1.json | 234 + .../RequestValidationSchemaV2.json | 456 + .../controllers/DatasetList/DatasetList.ts | 44 + .../DatasetListValidationSchema.json | 31 +- .../DatasetMetricsController.ts | 38 + .../DatasetMetricsValidationSchema.json | 12 + .../controllers/DatasetRead/DatasetRead.ts | 114 + .../controllers/DatasetReset/DatasetReset.ts | 57 + .../DatasetResetValidationSchema.json | 38 + .../DatasetStatusTransition.ts | 252 + .../ReadyToPublishSchema.json | 518 + .../RequestValidationSchema.json | 52 + .../DatasetUpdate/DatasetUpdate.ts | 148 + .../DatasetUpdateValidationSchema.json | 453 + .../DeleteTemplateController.ts | 0 .../EventValidation/EventValidation.ts | 6 +- .../RequestValidationSchema.json | 0 .../GenerateDataSchema/GenerateDataSchema.ts | 137 + .../RequestValidationSchema.json | 56 + .../GenerateSignedURL/GenerateSignedURL.ts | 0 .../GenerateSignedURLValidationSchema.json | 0 .../ListTemplateValidationSchema.json | 0 .../ListTemplatesController.ts | 0 .../NotificationChannel/Notification.ts | 122 + .../QueryTemplate/QueryTemplateController.ts | 0 .../QueryTemplate/QueryTemplateHelpers.ts | 0 .../QueryTemplateValidationSchema.json | 0 .../QueryWrapper/SqlQueryWrapper.ts | 4 +- .../ReadTemplateController.ts | 0 .../UpdateTemplateController.ts | 0 .../UpdateTemplateValidationSchema.json | 0 .../exceptions/SchemaGenerationException.ts | 8 + .../{v2 => }/helpers/ErrorResponseHandler.ts | 0 .../src/{v2 => }/helpers/ResponseHandler.ts | 20 +- api-service/src/logger/index.ts | 6 + .../{v2 => }/metrics/prometheus/entities.ts | 0 .../{v2 => }/metrics/prometheus/helpers.ts | 30 +- .../src/{v2 => }/metrics/prometheus/index.ts | 2 +- .../{v2 => }/metrics/prometheus/metrics.ts | 0 .../src/middlewares/RBAC_middleware.ts | 179 + api-service/src/middlewares/errors.ts | 19 + .../middlewares/setDataToRequestObject.ts | 0 api-service/src/models/Alert.ts | 74 + api-service/src/models/ConnectorInstances.ts | 53 + api-service/src/models/ConnectorRegistry.ts | 85 + api-service/src/{v2 => }/models/Dataset.ts | 18 +- .../src/{v2 => }/models/DatasetDraft.ts | 22 +- .../{v2 => }/models/DatasetSourceConfig.ts | 0 .../models/DatasetSourceConfigDraft.ts | 5 +- api-service/src/{v2 => }/models/Datasource.ts | 5 + .../src/{v2 => }/models/DatasourceDraft.ts | 8 +- api-service/src/models/Metric.ts | 30 + api-service/src/models/Notification.ts | 46 + .../src/{v2 => }/models/QueryTemplate.ts | 0 api-service/src/models/Silence.ts | 38 + .../src/{v2 => }/models/Transformation.ts | 0 .../{v2 => }/models/TransformationDraft.ts | 4 - api-service/src/routes/AlertsRouter.ts | 41 + api-service/src/routes/DruidProxyRouter.ts | 23 + .../MetricRouter.ts} | 4 +- api-service/src/routes/Router.ts | 65 + api-service/src/services/CipherService.ts | 30 + .../CloudServices/AWSStorageService.ts | 42 +- .../CloudServices/AzureStorageService.ts | 2 +- .../CloudServices/GCPStorageService.ts | 2 +- .../{v2 => }/services/CloudServices/index.ts | 0 .../{v2 => }/services/CloudServices/types.ts | 0 api-service/src/services/ConnectorService.ts | 15 + .../src/services/DatasetHealthService.ts | 513 + api-service/src/services/DatasetService.ts | 395 + .../services/DatasetSourceConfigService.ts | 0 api-service/src/services/DatasourceService.ts | 23 + api-service/src/services/HealthService.ts | 20 + .../{v2 => }/services/QueryTemplateService.ts | 0 .../SchemaGenerateService/ConfigSuggester.ts | 61 + .../SchemaGenerateService/Constants.json | 20 + .../DataSchemaService.ts | 106 + .../SchemaGenerateService/SchemaAnalyser.ts | 193 + .../SchemaArrayValidator.ts | 52 + .../SchemaCardinalityAnalyser.ts | 50 + .../SchemaGeneratorUtils.ts | 50 + .../SchemaGenerateService/SchemaHandler.ts | 168 + .../SchemaGenerateService/SchemaMapping.json | 103 + .../SchemaGenerateService/SchemaMerger.ts | 18 + .../SuggestionTemplate.ts | 74 + .../SchemaGenerateService/Template.ts | 107 + api-service/src/services/SystemConfig.ts | 22 + api-service/src/services/TableGenerator.ts | 298 + .../{v2 => }/services/ValidationService.ts | 0 .../src/{v1 => }/services/WrapperService.ts | 6 +- api-service/src/services/fs.ts | 18 + .../src/services/managers/constants.ts | 32 + .../managers/grafana/alert/helpers/index.ts | 302 + .../services/managers/grafana/alert/index.ts | 114 + .../src/services/managers/grafana/index.ts | 6 + .../grafana/notification/channels/email.ts | 53 + .../grafana/notification/channels/index.ts | 11 + .../grafana/notification/channels/slack.ts | 50 + .../grafana/notification/channels/teams.ts | 47 + .../grafana/notification/helpers/index.ts | 56 + .../managers/grafana/notification/index.ts | 26 + .../grafana/notification/templates/index.ts | 9 + .../grafana/silences/helpers/index.ts | 50 + .../managers/grafana/silences/index.ts | 43 + api-service/src/services/managers/index.ts | 217 + .../managers/prometheus/alert/index.ts | 17 + .../src/services/managers/prometheus/index.ts | 5 + .../managers/prometheus/notification/index.ts | 11 + .../managers/prometheus/silences/index.ts | 24 + .../src/{v2 => }/services/telemetry.ts | 12 +- .../data => telemetry}/telemetryActions.ts | 5 +- .../ConnectorsList/ConnectorsList.spec.ts | 132 + .../Connectors/ConnectorsList/Fixtures.ts | 298 + .../ConnectorsRead/ConnectorsRead.spec.ts | 117 + .../Connectors/ConnectorsRead/Fixtures.ts | 284 + .../DataIngestTest/DataIngestionTest.spec.ts | 74 +- .../DataIngestTest/Fixtures.ts | 0 .../DataOutTest/DataQueryTest.spec.ts | 16 +- .../DatasetManagement/DataOutTest/Fixtures.ts | 23 + .../DatasetCreate/DatasetCreate.spec.ts | 95 +- .../DatasetCreate/Fixtures.ts | 290 +- .../DatasetList/DatasetList.spec.ts | 119 + .../DatasetManagement/DatasetList/Fixtures.ts | 96 + .../DatasetRead/DatasetRead.spec.ts | 355 + .../DatasetManagement/DatasetRead/Fixtures.ts | 143 + .../DatasetDelete.spec.ts | 86 + .../DatasetLive.spec.ts | 336 + .../DatasetReadyToPublish.spec.ts | 133 + .../DatasetRetire.spec.ts | 234 + .../DatasetStatusTransition.spec.ts | 60 + .../DatasetStatusTransition/Fixtures.ts | 164 + .../DatasetUpdate/DatasetConnectors.spec.ts | 74 + .../DatasetUpdate/DatasetDedup.spec.ts | 26 +- .../DatasetUpdate/DatasetDenorm.spec.ts} | 105 +- .../DatasetUpdate/DatasetExtraction.spec.ts | 39 +- .../DatasetUpdate/DatasetTags.spec.ts | 77 + .../DatasetTransformation.spec.ts | 102 + .../DatasetUpdate/DatasetUpdate.spec.ts | 195 +- .../DatasetUpdate/DatasetValidation.spec.ts | 27 +- .../DatasetUpdate/Fixtures.ts | 258 +- .../GenerateSignedURL/Fixtures.ts | 16 +- .../GenerateSignedURL.spec.ts | 4 +- .../CreateTemplate/CreateTemplate.spec.ts | 6 +- .../QueryTemplates/CreateTemplate/Fixtures.ts | 0 .../DeleteTemplate/DeleteTemplate.spec.ts | 8 +- .../QueryTemplates/ListTemplates/Fixtures.ts | 0 .../ListTemplates/ListTemplates.spec.ts | 6 +- .../ReadTemplate/ReadTemplate.spec.ts | 18 +- .../TemplateQuerying/Fixtures.ts | 0 .../TemplateQuerying/TemplateQuerying.spec.ts | 42 +- .../QueryTemplates/UpdateTemplate/Fixtures.ts | 0 .../UpdateTemplate/UpdateTemplate.spec.ts | 6 +- .../tests/QueryWrapper/SqlWrapper/Fixtures.ts | 0 .../SqlWrapper/SqlWrapper.spec.ts | 6 +- api-service/src/types/AlertModels.ts | 36 + .../src/{v2 => }/types/ConfigModels.ts | 16 + .../{v1/models => types}/ConnectionModels.ts | 0 .../src/{v2 => }/types/DatasetModels.ts | 16 +- .../src/{v1/models => types}/ExhaustModels.ts | 0 .../src/{v2 => }/types/IngestionModels.ts | 12 + api-service/src/{v2 => }/types/MetricModel.ts | 2 +- api-service/src/types/ObsrvError.ts | 24 + .../src/{v1/models => types}/QueryModels.ts | 0 .../src/{v2 => }/types/ResponseModel.ts | 0 .../src/{v2 => }/types/SampleURLModel.ts | 0 api-service/src/types/SchemaModel.ts | 115 + .../{v1/models => types}/ValidationModels.ts | 0 api-service/src/{v2 => }/utils/common.ts | 0 api-service/src/v1/configs/Config.ts | 109 - api-service/src/v1/configs/Extensions.ts | 11 - api-service/src/v1/configs/RoutesConfig.ts | 181 - api-service/src/v1/connectors/DbConnector.ts | 179 - .../src/v1/connectors/HttpConnector.ts | 28 - .../src/v1/connectors/KafkaConnector.ts | 25 - api-service/src/v1/generators/SchemaMerger.ts | 7 - api-service/src/v1/helpers/DatasetConfigs.ts | 19 - .../src/v1/helpers/DatasetSourceConfigs.ts | 52 - api-service/src/v1/helpers/Datasets.ts | 93 - api-service/src/v1/helpers/Datasources.ts | 65 - api-service/src/v1/helpers/DbUtil.ts | 60 - .../src/v1/helpers/ErrorResponseHandler.ts | 32 - api-service/src/v1/helpers/LakehouseUtil.ts | 51 - api-service/src/v1/helpers/ResponseHandler.ts | 48 - .../src/v1/helpers/ValidationService.ts | 12 - .../AWSStorageService.js | 475 - .../AzureStorageService.js | 405 - .../BaseStorageService.js | 109 - .../GCPStorageService.js | 278 - .../src/v1/lib/client-cloud-services/index.js | 34 - .../client-cloud-services/storageLogger.js | 56 - .../src/v1/lib/dispatcher/dispatcher.js | 54 - .../src/v1/lib/dispatcher/kafka-dispatcher.js | 60 - api-service/src/v1/lib/envVariables.js | 15 - .../src/v1/lib/services/TelemetryService.js | 124 - api-service/src/v1/managers/Extensions.ts | 28 - api-service/src/v1/models/ConfigModels.ts | 44 - api-service/src/v1/models/DatasetModels.ts | 57 - api-service/src/v1/models/IngestionModels.ts | 35 - api-service/src/v1/resources/Constants.json | 111 - .../v1/resources/schemas/DataExhaustReq.json | 21 - .../resources/schemas/DataIngestionReq.json | 11 - .../resources/schemas/DatasetConfigDefault.ts | 87 - .../resources/schemas/DatasetCreateReq.json | 99 - .../v1/resources/schemas/DatasetListReq.json | 9 - .../schemas/DatasetSourceConfigSaveReq.json | 33 - .../schemas/DatasetSourceConfigUpdateReq.json | 33 - .../resources/schemas/DatasetUpdateReq.json | 99 - .../schemas/DatasourceConfigDefault.json | 22 - .../resources/schemas/DatasourceSaveReq.json | 128 - .../schemas/DatasourceUpdateReq.json | 130 - .../v1/resources/schemas/QueryRequest.json | 49 - .../resources/schemas/SchemaValidatorReq.json | 30 - .../resources/schemas/SubmitIngestionReq.json | 12 - api-service/src/v1/routes/Router.ts | 80 - .../src/v1/services/ClientCloudService.ts | 74 - .../src/v1/services/DataSourceService.ts | 81 - api-service/src/v1/services/DatasetService.ts | 58 - .../v1/services/DatasetSourceConfigService.ts | 53 - .../services/EventsValidationAgainstSchema.ts | 32 - api-service/src/v1/services/HealthService.ts | 46 - .../src/v1/services/IngestorService.ts | 105 - api-service/src/v1/services/QueryService.ts | 49 - .../src/v1/services/ValidationService.ts | 37 - api-service/src/v1/services/telemetry.ts | 187 - .../src/v1/test/ClientCloudService.spec.ts | 128 - api-service/src/v1/test/Config.ts | 31 - .../DatasetSourceConfigTestService.spec.ts | 300 - .../src/v1/test/DatasetTestService.spec.ts | 366 - .../src/v1/test/DatasourceTestService.spec.ts | 493 - api-service/src/v1/test/Fixtures.ts | 154 - .../src/v1/test/IngestorTestService.spec.ts | 276 - .../src/v1/test/QueryTestService.spec.ts | 738 - api-service/src/v1/test/TestConnector.spec.ts | 193 - api-service/src/v1/utils/common.ts | 38 - .../src/v1/validators/QueryValidator.ts | 228 - .../src/v1/validators/RequestsValidator.ts | 87 - .../src/v2/configs/DatasetConfigDefault.ts | 92 - api-service/src/v2/configs/IngestionConfig.ts | 17 - api-service/src/v2/configs/QueryRules.ts | 56 - .../connections/commandServiceConnection.ts | 2 - .../controllers/DataOut/DataOutController.ts | 56 - .../DatasetCreate/DatasetCreate.ts | 175 - .../DatasetCreateValidationSchema.json | 214 - .../v2/controllers/DatasetList/DatasetList.ts | 146 - .../v2/controllers/DatasetRead/DatasetRead.ts | 119 - .../DatasetUpdate/DatasetUpdate.ts | 408 - api-service/src/v2/logger/index.ts | 15 - api-service/src/v2/routes/Router.ts | 43 - api-service/src/v2/services/DatasetService.ts | 70 - .../src/v2/services/DatasourceService.ts | 31 - api-service/src/v2/services/HealthService.ts | 45 - .../src/v2/services/IngestionService.ts | 260 - .../src/v2/telemetry/telemetryActions.ts | 21 - .../DatasetManagement/DataOutTest/Fixtures.ts | 23 - .../DatasetList/DatasetList.spec.ts | 184 - .../DatasetManagement/DatasetList/Fixtures.ts | 234 - .../DatasetRead/DatasetRead.spec.ts | 157 - .../DatasetManagement/DatasetRead/Fixtures.ts | 167 - .../DatasetUpdate/DatasetDenorm.spec.ts | 213 - .../DatasetTransformation.spec.ts | 287 - api-service/src/v2/types/ConnectionModels.ts | 13 - api-service/src/v2/types/ExhaustModels.ts | 4 - api-service/src/v2/types/QueryModels.ts | 66 - api-service/src/v2/types/ValidationModels.ts | 6 - .../{v2 => }/validators/RequestsValidator.ts | 0 api-service/swagger-doc/openapi_v2.yml | 3191 +- .../swagger-doc/v2_updated_doc_openapi.yml | 4893 ++++ command-service/Dockerfile | 19 +- command-service/README.md | 9 +- command-service/docs/pii_swagger_doc.yml | 147 + .../connector-cron-jobs/Chart.yaml | 24 + .../templates/_helpers.tpl | 74 + .../templates/cronjob.yaml | 59 + .../connector-cron-jobs/templates/rbac.yaml | 48 + .../connector-cron-jobs/values.yaml | 29 + .../helm-charts/flink-connector/Chart.lock | 18 + .../helm-charts/flink-connector/Chart.yaml | 7 + .../flink-connector/charts/.helmignore | 23 + .../flink-connector/charts/common/Chart.yaml | 12 + .../charts/common/templates/_affinities.tpl | 139 + .../charts/common/templates/_capabilities.tpl | 229 + .../charts/common/templates/_configs.tpl | 0 .../charts/common/templates/_errors.tpl | 28 + .../charts/common/templates/_images.tpl | 117 + .../charts/common/templates/_ingress.tpl | 73 + .../charts/common/templates/_labels.tpl | 46 + .../charts/common/templates/_names.tpl | 71 + .../charts/common/templates/_secrets.tpl | 172 + .../charts/common/templates/_storage.tpl | 28 + .../charts/common/templates/_tplvalues.tpl | 38 + .../charts/common/templates/_utils.tpl | 77 + .../charts/common/templates/_variables.tpl | 41 + .../charts/common/templates/_warnings.tpl | 19 + .../templates/validations/_cassandra.tpl | 77 + .../common/templates/validations/_mariadb.tpl | 108 + .../common/templates/validations/_mongodb.tpl | 113 + .../common/templates/validations/_mysql.tpl | 108 + .../templates/validations/_postgresql.tpl | 134 + .../common/templates/validations/_redis.tpl | 81 + .../templates/validations/_validations.tpl | 51 + .../flink-connector/charts/common/values.yaml | 82 + .../flink-connector/templates/NOTES.txt | 0 .../templates/_base_serviceAccount.tpl | 4 + .../flink-connector/templates/_helpers.tpl | 0 .../templates/_image_flink.tpl | 16 + .../flink-connector/templates/_namespace.tpl | 13 + .../flink-connector/templates/configmap.yaml | 21 + .../flink-connector/templates/deployment.yaml | 238 + .../flink-connector/templates/hpa.yaml | 20 + .../flink-connector/templates/ingress.yaml | 24 + .../flink-connector/templates/service.yaml | 42 + .../templates/serviceaccount.yaml | 11 + .../templates/servicemonitor.yaml | 31 + .../helm-charts/flink-connector/values.yaml | 253 + .../spark-connector-cron/Chart.lock | 6 + .../spark-connector-cron/Chart.yaml | 11 + .../charts/common/.helmignore | 23 + .../charts/common/Chart.yaml | 12 + .../charts/common/templates/_affinities.tpl | 139 + .../charts/common/templates/_capabilities.tpl | 229 + .../charts/common/templates/_configs.tpl | 0 .../charts/common/templates/_errors.tpl | 28 + .../charts/common/templates/_images.tpl | 117 + .../charts/common/templates/_ingress.tpl | 73 + .../charts/common/templates/_labels.tpl | 46 + .../charts/common/templates/_names.tpl | 71 + .../charts/common/templates/_secrets.tpl | 172 + .../charts/common/templates/_storage.tpl | 28 + .../charts/common/templates/_tplvalues.tpl | 38 + .../charts/common/templates/_utils.tpl | 77 + .../charts/common/templates/_variables.tpl | 41 + .../charts/common/templates/_warnings.tpl | 19 + .../templates/validations/_cassandra.tpl | 77 + .../common/templates/validations/_mariadb.tpl | 108 + .../common/templates/validations/_mongodb.tpl | 113 + .../common/templates/validations/_mysql.tpl | 108 + .../templates/validations/_postgresql.tpl | 134 + .../common/templates/validations/_redis.tpl | 81 + .../templates/validations/_validations.tpl | 51 + .../charts/common/values.yaml | 82 + .../spark-connector-cron/templates/NOTES.txt | 0 .../templates/_base_serviceAccount.tpl | 4 + .../templates/_cron_release_name.tpl | 4 + .../spark-connector-cron/templates/_image.tpl | 10 + .../templates/_namespace.tpl | 14 + .../templates/cronjob.yaml | 83 + .../spark-connector-cron/values.yaml | 153 + command-service/requirements.txt | 15 +- command-service/src/command/__init__.py | 6 +- .../src/command/alert_manager_command.py | 215 + .../src/command/command_executor.py | 104 +- .../src/command/connector_command.py | 370 + .../src/command/connector_registry.py | 563 + .../src/command/dataset_command.py | 106 + command-service/src/command/db_command.py | 429 + command-service/src/command/druid_command.py | 53 + command-service/src/command/flink_command.py | 60 +- command-service/src/command/icommand.py | 2 +- .../src/command/telemetry_command.py | 59 + command-service/src/config/__init__.py | 3 +- command-service/src/config/config.py | 19 +- command-service/src/config/pii_rules.yml | 118 + command-service/src/config/service_config.yml | 188 +- command-service/src/exception/exception.py | 10 +- command-service/src/metrics/__init__.py | 3 + command-service/src/metrics/helper.py | 38 + command-service/src/metrics/metrics.py | 41 + command-service/src/model/__init__.py | 11 +- command-service/src/model/data_models.py | 109 +- command-service/src/model/db_models.py | 127 + command-service/src/model/telemetry_models.py | 75 + command-service/src/routes.py | 242 +- command-service/src/service/__init__.py | 4 +- command-service/src/service/db_service.py | 73 + .../src/service/detect_pii_service.py | 33 + command-service/src/service/http_service.py | 39 +- command-service/src/service/re_pii_model.py | 64 + .../src/service/telemetry_service.py | 38 + command-service/src/stubs/snippets.py | 13 + 423 files changed, 30607 insertions(+), 36277 deletions(-) delete mode 100644 api-service/package-lock.json create mode 100644 api-service/postman-collection/updated_v2_collection.json rename api-service/src/{v2 => }/configs/Config.ts (76%) rename api-service/src/{v2 => }/configs/ConnectionsConfig.ts (100%) create mode 100644 api-service/src/configs/DatasetConfigDefault.ts create mode 100644 api-service/src/configs/IngestionConfig.ts rename api-service/src/{v1 => }/configs/QueryRules.ts (100%) create mode 100644 api-service/src/connections/commandServiceConnection.ts rename api-service/src/{v2 => }/connections/databaseConnection.ts (70%) rename api-service/src/{v2 => }/connections/druidConnection.ts (84%) create mode 100644 api-service/src/connections/grafanaConnection.ts rename api-service/src/{v2 => }/connections/kafkaConnection.ts (79%) create mode 100644 api-service/src/controllers/Alerts/Alerts.ts create mode 100644 api-service/src/controllers/Alerts/Metric.ts create mode 100644 api-service/src/controllers/Alerts/Silence.ts create mode 100644 api-service/src/controllers/ConnectorsList/ConnectorsList.ts create mode 100644 api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json create mode 100644 api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts rename api-service/src/{v2 => }/controllers/CreateQueryTemplate/CreateTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/CreateQueryTemplate/QueryTemplateValidator.ts (100%) rename api-service/src/{v2 => }/controllers/DataExhaust/DataExhaustController.ts (94%) rename api-service/src/{v2 => }/controllers/DataIngestion/DataIngestionController.ts (67%) rename api-service/src/{v2 => }/controllers/DataIngestion/validationSchema.json (100%) create mode 100644 api-service/src/controllers/DataOut/DataOutController.ts rename api-service/src/{v2 => }/controllers/DataOut/DataOutValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DataOut/QueryRules.ts (100%) rename api-service/src/{v2 => }/controllers/DataOut/QueryValidator.ts (100%) create mode 100644 api-service/src/controllers/DatasetCopy/DatasetCopy.ts create mode 100644 api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts create mode 100644 api-service/src/controllers/DatasetCopy/RequestValidationSchema.json create mode 100644 api-service/src/controllers/DatasetCreate/DatasetCreate.ts rename api-service/src/{v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json => controllers/DatasetCreate/DatasetCreateValidationSchema.json} (52%) create mode 100644 api-service/src/controllers/DatasetExport/DatasetExport.ts create mode 100644 api-service/src/controllers/DatasetHealth/DatasetHealth.ts create mode 100644 api-service/src/controllers/DatasetHealth/DatasetHealthValidationSchema.json create mode 100644 api-service/src/controllers/DatasetImport/DatasetImport.ts create mode 100644 api-service/src/controllers/DatasetImport/DatasetImportHelper.ts create mode 100644 api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json create mode 100644 api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json create mode 100644 api-service/src/controllers/DatasetList/DatasetList.ts rename api-service/src/{v2 => }/controllers/DatasetList/DatasetListValidationSchema.json (61%) create mode 100644 api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts create mode 100644 api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json create mode 100644 api-service/src/controllers/DatasetRead/DatasetRead.ts create mode 100644 api-service/src/controllers/DatasetReset/DatasetReset.ts create mode 100644 api-service/src/controllers/DatasetReset/DatasetResetValidationSchema.json create mode 100644 api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts create mode 100644 api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json create mode 100644 api-service/src/controllers/DatasetStatusTransition/RequestValidationSchema.json create mode 100644 api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts create mode 100644 api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json rename api-service/src/{v2 => }/controllers/DeleteQueryTemplate/DeleteTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/EventValidation/EventValidation.ts (91%) rename api-service/src/{v2 => }/controllers/EventValidation/RequestValidationSchema.json (100%) create mode 100644 api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts create mode 100644 api-service/src/controllers/GenerateDataSchema/RequestValidationSchema.json rename api-service/src/{v2 => }/controllers/GenerateSignedURL/GenerateSignedURL.ts (100%) rename api-service/src/{v2 => }/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/ListQueryTemplates/ListTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/ListQueryTemplates/ListTemplatesController.ts (100%) create mode 100644 api-service/src/controllers/NotificationChannel/Notification.ts rename api-service/src/{v2 => }/controllers/QueryTemplate/QueryTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/QueryTemplate/QueryTemplateHelpers.ts (100%) rename api-service/src/{v2 => }/controllers/QueryTemplate/QueryTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/QueryWrapper/SqlQueryWrapper.ts (89%) rename api-service/src/{v2 => }/controllers/ReadQueryTemplate/ReadTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/UpdateQueryTemplate/UpdateTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json (100%) create mode 100644 api-service/src/exceptions/SchemaGenerationException.ts rename api-service/src/{v2 => }/helpers/ErrorResponseHandler.ts (100%) rename api-service/src/{v2 => }/helpers/ResponseHandler.ts (64%) create mode 100644 api-service/src/logger/index.ts rename api-service/src/{v2 => }/metrics/prometheus/entities.ts (100%) rename api-service/src/{v2 => }/metrics/prometheus/helpers.ts (54%) rename api-service/src/{v2 => }/metrics/prometheus/index.ts (95%) rename api-service/src/{v2 => }/metrics/prometheus/metrics.ts (100%) create mode 100644 api-service/src/middlewares/RBAC_middleware.ts create mode 100644 api-service/src/middlewares/errors.ts rename api-service/src/{v2 => }/middlewares/setDataToRequestObject.ts (100%) create mode 100644 api-service/src/models/Alert.ts create mode 100644 api-service/src/models/ConnectorInstances.ts create mode 100644 api-service/src/models/ConnectorRegistry.ts rename api-service/src/{v2 => }/models/Dataset.ts (78%) rename api-service/src/{v2 => }/models/DatasetDraft.ts (85%) rename api-service/src/{v2 => }/models/DatasetSourceConfig.ts (100%) rename api-service/src/{v2 => }/models/DatasetSourceConfigDraft.ts (90%) rename api-service/src/{v2 => }/models/Datasource.ts (92%) rename api-service/src/{v2 => }/models/DatasourceDraft.ts (93%) create mode 100644 api-service/src/models/Metric.ts create mode 100644 api-service/src/models/Notification.ts rename api-service/src/{v2 => }/models/QueryTemplate.ts (100%) create mode 100644 api-service/src/models/Silence.ts rename api-service/src/{v2 => }/models/Transformation.ts (100%) rename api-service/src/{v2 => }/models/TransformationDraft.ts (93%) create mode 100644 api-service/src/routes/AlertsRouter.ts create mode 100644 api-service/src/routes/DruidProxyRouter.ts rename api-service/src/{v2/routes/metricRouter.ts => routes/MetricRouter.ts} (56%) create mode 100644 api-service/src/routes/Router.ts create mode 100644 api-service/src/services/CipherService.ts rename api-service/src/{v2 => }/services/CloudServices/AWSStorageService.ts (75%) rename api-service/src/{v2 => }/services/CloudServices/AzureStorageService.ts (99%) rename api-service/src/{v2 => }/services/CloudServices/GCPStorageService.ts (98%) rename api-service/src/{v2 => }/services/CloudServices/index.ts (100%) rename api-service/src/{v2 => }/services/CloudServices/types.ts (100%) create mode 100644 api-service/src/services/ConnectorService.ts create mode 100644 api-service/src/services/DatasetHealthService.ts create mode 100644 api-service/src/services/DatasetService.ts rename api-service/src/{v2 => }/services/DatasetSourceConfigService.ts (100%) create mode 100644 api-service/src/services/DatasourceService.ts create mode 100644 api-service/src/services/HealthService.ts rename api-service/src/{v2 => }/services/QueryTemplateService.ts (100%) create mode 100644 api-service/src/services/SchemaGenerateService/ConfigSuggester.ts create mode 100644 api-service/src/services/SchemaGenerateService/Constants.json create mode 100644 api-service/src/services/SchemaGenerateService/DataSchemaService.ts create mode 100644 api-service/src/services/SchemaGenerateService/SchemaAnalyser.ts create mode 100644 api-service/src/services/SchemaGenerateService/SchemaArrayValidator.ts create mode 100644 api-service/src/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts create mode 100644 api-service/src/services/SchemaGenerateService/SchemaGeneratorUtils.ts create mode 100644 api-service/src/services/SchemaGenerateService/SchemaHandler.ts create mode 100644 api-service/src/services/SchemaGenerateService/SchemaMapping.json create mode 100644 api-service/src/services/SchemaGenerateService/SchemaMerger.ts create mode 100644 api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts create mode 100644 api-service/src/services/SchemaGenerateService/Template.ts create mode 100644 api-service/src/services/SystemConfig.ts create mode 100644 api-service/src/services/TableGenerator.ts rename api-service/src/{v2 => }/services/ValidationService.ts (100%) rename api-service/src/{v1 => }/services/WrapperService.ts (98%) create mode 100644 api-service/src/services/fs.ts create mode 100644 api-service/src/services/managers/constants.ts create mode 100644 api-service/src/services/managers/grafana/alert/helpers/index.ts create mode 100644 api-service/src/services/managers/grafana/alert/index.ts create mode 100644 api-service/src/services/managers/grafana/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/email.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/slack.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/teams.ts create mode 100644 api-service/src/services/managers/grafana/notification/helpers/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/templates/index.ts create mode 100644 api-service/src/services/managers/grafana/silences/helpers/index.ts create mode 100644 api-service/src/services/managers/grafana/silences/index.ts create mode 100644 api-service/src/services/managers/index.ts create mode 100644 api-service/src/services/managers/prometheus/alert/index.ts create mode 100644 api-service/src/services/managers/prometheus/index.ts create mode 100644 api-service/src/services/managers/prometheus/notification/index.ts create mode 100644 api-service/src/services/managers/prometheus/silences/index.ts rename api-service/src/{v2 => }/services/telemetry.ts (91%) rename api-service/src/{v1/data => telemetry}/telemetryActions.ts (84%) create mode 100644 api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts create mode 100644 api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts create mode 100644 api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts create mode 100644 api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts rename api-service/src/{v2 => }/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts (77%) rename api-service/src/{v2 => }/tests/DatasetManagement/DataIngestTest/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts (95%) create mode 100644 api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts (54%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetCreate/Fixtures.ts (73%) create mode 100644 api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetList/Fixtures.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts (83%) rename api-service/src/{v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts => tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts} (52%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts (82%) create mode 100644 api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts create mode 100644 api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts (67%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts (84%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/Fixtures.ts (63%) rename api-service/src/{v2 => }/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts (78%) rename api-service/src/{v2 => }/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts (98%) rename api-service/src/{v2 => }/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts (98%) rename api-service/src/{v2 => }/tests/QueryTemplates/CreateTemplate/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts (94%) rename api-service/src/{v2 => }/tests/QueryTemplates/ListTemplates/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts (98%) rename api-service/src/{v2 => }/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts (86%) rename api-service/src/{v2 => }/tests/QueryTemplates/TemplateQuerying/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts (82%) rename api-service/src/{v2 => }/tests/QueryTemplates/UpdateTemplate/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts (98%) rename api-service/src/{v2 => }/tests/QueryWrapper/SqlWrapper/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts (95%) create mode 100644 api-service/src/types/AlertModels.ts rename api-service/src/{v2 => }/types/ConfigModels.ts (63%) rename api-service/src/{v1/models => types}/ConnectionModels.ts (100%) rename api-service/src/{v2 => }/types/DatasetModels.ts (81%) rename api-service/src/{v1/models => types}/ExhaustModels.ts (100%) rename api-service/src/{v2 => }/types/IngestionModels.ts (83%) rename api-service/src/{v2 => }/types/MetricModel.ts (90%) create mode 100644 api-service/src/types/ObsrvError.ts rename api-service/src/{v1/models => types}/QueryModels.ts (100%) rename api-service/src/{v2 => }/types/ResponseModel.ts (100%) rename api-service/src/{v2 => }/types/SampleURLModel.ts (100%) create mode 100644 api-service/src/types/SchemaModel.ts rename api-service/src/{v1/models => types}/ValidationModels.ts (100%) rename api-service/src/{v2 => }/utils/common.ts (100%) delete mode 100644 api-service/src/v1/configs/Config.ts delete mode 100644 api-service/src/v1/configs/Extensions.ts delete mode 100644 api-service/src/v1/configs/RoutesConfig.ts delete mode 100644 api-service/src/v1/connectors/DbConnector.ts delete mode 100644 api-service/src/v1/connectors/HttpConnector.ts delete mode 100644 api-service/src/v1/connectors/KafkaConnector.ts delete mode 100644 api-service/src/v1/generators/SchemaMerger.ts delete mode 100644 api-service/src/v1/helpers/DatasetConfigs.ts delete mode 100644 api-service/src/v1/helpers/DatasetSourceConfigs.ts delete mode 100644 api-service/src/v1/helpers/Datasets.ts delete mode 100644 api-service/src/v1/helpers/Datasources.ts delete mode 100644 api-service/src/v1/helpers/DbUtil.ts delete mode 100644 api-service/src/v1/helpers/ErrorResponseHandler.ts delete mode 100644 api-service/src/v1/helpers/LakehouseUtil.ts delete mode 100644 api-service/src/v1/helpers/ResponseHandler.ts delete mode 100644 api-service/src/v1/helpers/ValidationService.ts delete mode 100644 api-service/src/v1/lib/client-cloud-services/AWSStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/AzureStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/BaseStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/GCPStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/index.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/storageLogger.js delete mode 100644 api-service/src/v1/lib/dispatcher/dispatcher.js delete mode 100644 api-service/src/v1/lib/dispatcher/kafka-dispatcher.js delete mode 100644 api-service/src/v1/lib/envVariables.js delete mode 100644 api-service/src/v1/lib/services/TelemetryService.js delete mode 100644 api-service/src/v1/managers/Extensions.ts delete mode 100644 api-service/src/v1/models/ConfigModels.ts delete mode 100644 api-service/src/v1/models/DatasetModels.ts delete mode 100644 api-service/src/v1/models/IngestionModels.ts delete mode 100644 api-service/src/v1/resources/Constants.json delete mode 100644 api-service/src/v1/resources/schemas/DataExhaustReq.json delete mode 100644 api-service/src/v1/resources/schemas/DataIngestionReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetConfigDefault.ts delete mode 100644 api-service/src/v1/resources/schemas/DatasetCreateReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetListReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetUpdateReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasourceConfigDefault.json delete mode 100644 api-service/src/v1/resources/schemas/DatasourceSaveReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasourceUpdateReq.json delete mode 100644 api-service/src/v1/resources/schemas/QueryRequest.json delete mode 100644 api-service/src/v1/resources/schemas/SchemaValidatorReq.json delete mode 100644 api-service/src/v1/resources/schemas/SubmitIngestionReq.json delete mode 100644 api-service/src/v1/routes/Router.ts delete mode 100644 api-service/src/v1/services/ClientCloudService.ts delete mode 100644 api-service/src/v1/services/DataSourceService.ts delete mode 100644 api-service/src/v1/services/DatasetService.ts delete mode 100644 api-service/src/v1/services/DatasetSourceConfigService.ts delete mode 100644 api-service/src/v1/services/EventsValidationAgainstSchema.ts delete mode 100644 api-service/src/v1/services/HealthService.ts delete mode 100644 api-service/src/v1/services/IngestorService.ts delete mode 100644 api-service/src/v1/services/QueryService.ts delete mode 100644 api-service/src/v1/services/ValidationService.ts delete mode 100644 api-service/src/v1/services/telemetry.ts delete mode 100644 api-service/src/v1/test/ClientCloudService.spec.ts delete mode 100644 api-service/src/v1/test/Config.ts delete mode 100644 api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts delete mode 100644 api-service/src/v1/test/DatasetTestService.spec.ts delete mode 100644 api-service/src/v1/test/DatasourceTestService.spec.ts delete mode 100644 api-service/src/v1/test/Fixtures.ts delete mode 100644 api-service/src/v1/test/IngestorTestService.spec.ts delete mode 100644 api-service/src/v1/test/QueryTestService.spec.ts delete mode 100644 api-service/src/v1/test/TestConnector.spec.ts delete mode 100644 api-service/src/v1/utils/common.ts delete mode 100644 api-service/src/v1/validators/QueryValidator.ts delete mode 100644 api-service/src/v1/validators/RequestsValidator.ts delete mode 100644 api-service/src/v2/configs/DatasetConfigDefault.ts delete mode 100644 api-service/src/v2/configs/IngestionConfig.ts delete mode 100644 api-service/src/v2/configs/QueryRules.ts delete mode 100644 api-service/src/v2/connections/commandServiceConnection.ts delete mode 100644 api-service/src/v2/controllers/DataOut/DataOutController.ts delete mode 100644 api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts delete mode 100644 api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json delete mode 100644 api-service/src/v2/controllers/DatasetList/DatasetList.ts delete mode 100644 api-service/src/v2/controllers/DatasetRead/DatasetRead.ts delete mode 100644 api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts delete mode 100644 api-service/src/v2/logger/index.ts delete mode 100644 api-service/src/v2/routes/Router.ts delete mode 100644 api-service/src/v2/services/DatasetService.ts delete mode 100644 api-service/src/v2/services/DatasourceService.ts delete mode 100644 api-service/src/v2/services/HealthService.ts delete mode 100644 api-service/src/v2/services/IngestionService.ts delete mode 100644 api-service/src/v2/telemetry/telemetryActions.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DataOutTest/Fixtures.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts delete mode 100644 api-service/src/v2/types/ConnectionModels.ts delete mode 100644 api-service/src/v2/types/ExhaustModels.ts delete mode 100644 api-service/src/v2/types/QueryModels.ts delete mode 100644 api-service/src/v2/types/ValidationModels.ts rename api-service/src/{v2 => }/validators/RequestsValidator.ts (100%) create mode 100644 api-service/swagger-doc/v2_updated_doc_openapi.yml create mode 100644 command-service/docs/pii_swagger_doc.yml create mode 100644 command-service/helm-charts/connector-cron-jobs/Chart.yaml create mode 100644 command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl create mode 100644 command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml create mode 100644 command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml create mode 100644 command-service/helm-charts/connector-cron-jobs/values.yaml create mode 100644 command-service/helm-charts/flink-connector/Chart.lock create mode 100644 command-service/helm-charts/flink-connector/Chart.yaml create mode 100644 command-service/helm-charts/flink-connector/charts/.helmignore create mode 100644 command-service/helm-charts/flink-connector/charts/common/Chart.yaml create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl create mode 100644 command-service/helm-charts/flink-connector/charts/common/values.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/NOTES.txt create mode 100644 command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl create mode 100644 command-service/helm-charts/flink-connector/templates/_helpers.tpl create mode 100644 command-service/helm-charts/flink-connector/templates/_image_flink.tpl create mode 100644 command-service/helm-charts/flink-connector/templates/_namespace.tpl create mode 100644 command-service/helm-charts/flink-connector/templates/configmap.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/deployment.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/hpa.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/ingress.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/service.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/serviceaccount.yaml create mode 100644 command-service/helm-charts/flink-connector/templates/servicemonitor.yaml create mode 100644 command-service/helm-charts/flink-connector/values.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/Chart.lock create mode 100644 command-service/helm-charts/spark-connector-cron/Chart.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/.helmignore create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_configs.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/values.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/templates/NOTES.txt create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_image.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/values.yaml create mode 100644 command-service/src/command/alert_manager_command.py create mode 100644 command-service/src/command/connector_command.py create mode 100644 command-service/src/command/connector_registry.py create mode 100644 command-service/src/command/dataset_command.py create mode 100644 command-service/src/command/db_command.py create mode 100644 command-service/src/command/druid_command.py create mode 100644 command-service/src/command/telemetry_command.py create mode 100644 command-service/src/config/pii_rules.yml create mode 100644 command-service/src/metrics/__init__.py create mode 100644 command-service/src/metrics/helper.py create mode 100644 command-service/src/metrics/metrics.py create mode 100644 command-service/src/model/db_models.py create mode 100644 command-service/src/model/telemetry_models.py create mode 100644 command-service/src/service/db_service.py create mode 100644 command-service/src/service/detect_pii_service.py create mode 100644 command-service/src/service/re_pii_model.py create mode 100644 command-service/src/service/telemetry_service.py create mode 100644 command-service/src/stubs/snippets.py diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 12e967e2..8b4fc54d 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -24,5 +24,4 @@ jobs: cd api-service npm install npm run actions:test - npm run actions:test:v2 npm run lint \ No newline at end of file diff --git a/.gitignore b/.gitignore index aafda3a6..5691d838 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,16 @@ -.vscode -.idea +coverage +node_modules +.nyc_output +dist +credentials.js +uploads/ +.env +# Byte-compiled / optimized / DLL files +*.pyc +__pycache__/ +.vscode/ +*.DS_Store +#to ignore embedded postgres data files +data +connector-registry +package-lock.json diff --git a/api-service/.eslintignore b/api-service/.eslintignore index f56aadad..e528fbc0 100644 --- a/api-service/.eslintignore +++ b/api-service/.eslintignore @@ -4,6 +4,6 @@ docs coverage @types .nyc_output -src/v2/tests +src/tests src/v1 dist \ No newline at end of file diff --git a/api-service/.eslintrc b/api-service/.eslintrc index 598451ab..ca799240 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -9,7 +9,8 @@ "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], - "rules": { + "rules": { + "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], "@typescript-eslint/quotes": [ diff --git a/api-service/package-lock.json b/api-service/package-lock.json deleted file mode 100644 index e81a2537..00000000 --- a/api-service/package-lock.json +++ /dev/null @@ -1,24055 +0,0 @@ -{ - "name": "obsrv-api-service", - "version": "1.0.0", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "name": "obsrv-api-service", - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "@aws-sdk/client-s3": "^3.540.0", - "@aws-sdk/credential-providers": "^3.309.0", - "@aws-sdk/lib-storage": "^3.182.0", - "@aws-sdk/s3-request-presigner": "^3.540.0", - "@azure/storage-blob": "^12.17.0", - "@google-cloud/storage": "^7.9.0", - "@project-sunbird/logger": "^0.0.9", - "ajv": "^8.11.2", - "ajv-formats": "^2.1.1", - "aws-sdk": "^2.1348.0", - "axios": "^1.6.0", - "body-parser": "^1.20.2", - "compression": "^1.7.4", - "dateformat": "2.0.0", - "express": "^4.18.2", - "http-errors": "^2.0.0", - "http-status": "^1.5.3", - "kafka-node": "^5.0.0", - "kafkajs": "^2.2.4", - "kafkajs-snappy": "^1.1.0", - "kafkajs-snappy-typescript": "^1.0.3", - "knex": "^2.4.2", - "lodash": "^4.17.21", - "moment": "^2.29.4", - "multiparty": "4.2.1", - "node-sql-parser": "^5.1.0", - "pg": "^8.11.3", - "pg-hstore": "^2.3.4", - "prom-client": "^14.2.0", - "sequelize": "^6.37.1", - "slug": "^9.0.0", - "trino-client": "^0.2.2", - "uuid": "3.1.0", - "winston": "~2.4.3", - "winston-daily-rotate-file": "~3.2.1" - }, - "devDependencies": { - "@types/chai": "^4.3.3", - "@types/chai-as-promised": "^7.1.5", - "@types/chai-spies": "^1.0.3", - "@types/compression": "^1.7.2", - "@types/express": "^4.17.14", - "@types/http-errors": "^2.0.1", - "@types/kafkajs": "^1.9.0", - "@types/knex": "^0.16.1", - "@types/lodash": "^4.14.190", - "@types/mocha": "^10.0.0", - "@types/mock-knex": "^0.4.4", - "@types/moment": "^2.13.0", - "@types/node": "^18.11.9", - "@types/pg": "^8.6.6", - "@types/sinon": "^17.0.3", - "@types/slug": "^5.0.8", - "@types/uuid": "^9.0.1", - "@typescript-eslint/eslint-plugin": "^7.1.1", - "@typescript-eslint/parser": "^7.1.1", - "chai": "^4.3.6", - "chai-as-promised": "^7.1.1", - "chai-http": "^4.3.0", - "chai-spies": "^1.0.0", - "diff-json": "^2.0.0", - "eslint": "^8.57.0", - "mocha": "^10.1.0", - "nock": "^13.2.9", - "nodemon": "^3.0.1", - "nyc": "^15.1.0", - "sinon": "^17.0.1", - "ts-node": "^10.9.1", - "tsconfig-paths": "^4.1.0", - "typescript": "^4.8.4" - } - }, - "node_modules/@ampproject/remapping": { - "version": "2.2.0", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@aws-crypto/crc32": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", - "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", - "dependencies": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/crc32/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/crc32c": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz", - "integrity": "sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==", - "dependencies": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/crc32c/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/ie11-detection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", - "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", - "dependencies": { - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/ie11-detection/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/sha1-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz", - "integrity": "sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==", - "dependencies": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/sha256-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", - "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", - "dependencies": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/sha256-js": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/sha256-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", - "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", - "dependencies": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/sha256-js/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/supports-web-crypto": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", - "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", - "dependencies": { - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/util": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", - "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", - "dependencies": { - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/util/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-sdk/client-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.427.0.tgz", - "integrity": "sha512-9brRaNnl6haE7R3R43A5CSNw0k1YtB3xjuArbMg/p6NDUpvRSRgOVNWu2R02Yjh/j2ZuaLOCPLuCipb+PHQPKQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/client-s3": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.590.0.tgz", - "integrity": "sha512-so+pNua0ihsHaSdskw8HCwruoYTAfYSEs3ix4GD1++83C96KaJp3udAutYiCA+84JXg9zitFa7eK7ORJAVZmTw==", - "dependencies": { - "@aws-crypto/sha1-browser": "3.0.0", - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-bucket-endpoint": "3.587.0", - "@aws-sdk/middleware-expect-continue": "3.577.0", - "@aws-sdk/middleware-flexible-checksums": "3.587.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-location-constraint": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/middleware-signing": "3.587.0", - "@aws-sdk/middleware-ssec": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@aws-sdk/xml-builder": "3.575.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/eventstream-serde-browser": "^3.0.0", - "@smithy/eventstream-serde-config-resolver": "^3.0.0", - "@smithy/eventstream-serde-node": "^3.0.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-blob-browser": "^3.0.0", - "@smithy/hash-node": "^3.0.0", - "@smithy/hash-stream-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/md5-js": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "@smithy/util-utf8": "^3.0.0", - "@smithy/util-waiter": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.590.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "dependencies": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-signing": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.587.0.tgz", - "integrity": "sha512-tiZaTDj4RvhXGRAlncFn7CSEfL3iNPO67WSaxAq+Ls5j1VgczPhu5262cWONNoMgth3nXR1hhLC4ITSl/a6AzA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sso-oidc": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dependencies": { - "@smithy/types": "^3.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "dependencies": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dependencies": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@aws-sdk/client-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.427.0.tgz", - "integrity": "sha512-sFVFEmsQ1rmgYO1SgrOTxE/MTKpeE4hpOkm1WqhLQK7Ij136vXpjCxjH1JYZiHiUzO1wr9t4ex4dlB5J3VS/Xg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.590.0.tgz", - "integrity": "sha512-3yCLPjq6WFfDpdUJKk/gSz4eAPDTjVknXaveMPi2QoVBCshneOnJsV16uNKlpVF1frTHrrDRfKYmbaVh6nFBvQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.590.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "dependencies": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sso-oidc": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dependencies": { - "@smithy/types": "^3.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "dependencies": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dependencies": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@aws-sdk/client-sts": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.427.0.tgz", - "integrity": "sha512-le2wLJKILyWuRfPz2HbyaNtu5kEki+ojUkTqCU6FPDRrqUvEkaaCBH9Awo/2AtrCfRkiobop8RuTTj6cAnpiJg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-sdk-sts": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/core": { - "version": "3.588.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.588.0.tgz", - "integrity": "sha512-O1c2+9ce46Z+iiid+W3iC1IvPbfIo5ev9CBi54GdNB9SaI8/3+f8MJcux0D6c9toCF0ArMersN/gp8ek57e9uQ==", - "dependencies": { - "@smithy/core": "^2.1.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.427.0.tgz", - "integrity": "sha512-BQNzNrMJlBAfXhYNdAUqaVASpT9Aho5swj7glZKxx4Uds1w5Pih2e14JWgnl8XgUWAZ36pchTrV1aA4JT7N8vw==", - "dependencies": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.425.0.tgz", - "integrity": "sha512-J20etnLvMKXRVi5FK4F8yOCNm2RTaQn5psQTGdDEPWJNGxohcSpzzls8U2KcMyUJ+vItlrThr4qwgpHG3i/N0w==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.425.0.tgz", - "integrity": "sha512-aP9nkoVWf+OlNMecrUqe4+RuQrX13nucVbty0HTvuwfwJJj0T6ByWZzle+fo1D+5OxvJmtzTflBWt6jUERdHWA==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.427.0.tgz", - "integrity": "sha512-NmH1cO/w98CKMltYec3IrJIIco19wRjATFNiw83c+FGXZ+InJwReqBnruxIOmKTx2KDzd6fwU1HOewS7UjaaaQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.427.0.tgz", - "integrity": "sha512-wYYbQ57nKL8OfgRbl8k6uXcdnYml+p3LSSfDUAuUEp1HKlQ8lOXFJ3BdLr5qrk7LhpyppSRnWBmh2c3kWa7ANQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.425.0.tgz", - "integrity": "sha512-YY6tkLdvtb1Fgofp3b1UWO+5vwS14LJ/smGmuGpSba0V7gFJRdcrJ9bcb9vVgAGuMdjzRJ+bUKlLLtqXkaykEw==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.427.0.tgz", - "integrity": "sha512-c+tXyS/i49erHs4bAp6vKNYeYlyQ0VNMBgoco0LCn1rL0REtHbfhWMnqDLF6c2n3yIWDOTrQu0D73Idnpy16eA==", - "dependencies": { - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/token-providers": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.425.0.tgz", - "integrity": "sha512-/0R65TgRzL01JU3SzloivWNwdkbIhr06uY/F5pBHf/DynQqaspKNfdHn6AiozgSVDfwRHFjKBTUy6wvf3QFkuA==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.427.0.tgz", - "integrity": "sha512-rKKohSHju462vo+uQnPjcEZPBAfAMgGH6K1XyyCNpuOC0yYLkG87PYpvAQeb8riTrkHPX0dYUHuTHZ6zQgMGjA==", - "dependencies": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-cognito-identity": "3.427.0", - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-http": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/lib-storage": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.427.0.tgz", - "integrity": "sha512-JE26Zo4SMMY2SGlD/FilF5MpLuP8D2IrW+ye/J77dwh91gyGnNa/lubJ7WbVjIAxgeaDHsAkcpNO4VR5k6JCKg==", - "dependencies": { - "@smithy/abort-controller": "^2.0.1", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/smithy-client": "^2.1.9", - "buffer": "5.6.0", - "events": "3.3.0", - "stream-browserify": "3.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-s3": "^3.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.587.0.tgz", - "integrity": "sha512-HkFXLPl8pr6BH/Q0JpOESqEKL0ZK3sk7aSZ1S6GE4RXET7H5R94THULXqQFZzD48gZcyFooO/yNKZTqrZFaWKg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.577.0.tgz", - "integrity": "sha512-6dPp8Tv4F0of4un5IAyG6q++GrRrNQQ4P2NAMB1W0VO4JoEu1C8GievbbDLi88TFIFmtKpnHB0ODCzwnoe8JsA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.587.0.tgz", - "integrity": "sha512-URMwp/budDvKhIvZ4a6zIBfFTun/iDlPWXqsGKYjEtHt8jz27OSjCZtDtIeqW4WTBdKL8KZgQcl+DdaE5M1qiQ==", - "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@aws-crypto/crc32c": "3.0.0", - "@aws-sdk/types": "3.577.0", - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.425.0.tgz", - "integrity": "sha512-E5Gt41LObQ+cr8QnLthwsH3MtVSNXy1AKJMowDr85h0vzqA/FHUkgHyOGntgozzjXT5M0MaSRYxS0xwTR5D4Ew==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.577.0.tgz", - "integrity": "sha512-DKPTD2D2s+t2QUo/IXYtVa/6Un8GZ+phSTBkyBNx2kfZz4Kwavhl/JJzSqTV3GfCXkVdFu7CrjoX7BZ6qWeTUA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-logger": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.425.0.tgz", - "integrity": "sha512-INE9XWRXx2f4a/r2vOU0tAmgctVp7nEaEasemNtVBYhqbKLZvr9ndLBSgKGgJ8LIcXAoISipaMuFiqIGkFsm7A==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.425.0.tgz", - "integrity": "sha512-77gnzJ5b91bgD75L/ugpOyerx6lR3oyS4080X1YI58EzdyBMkDrHM4FbMcY2RynETi3lwXCFzLRyZjWXY1mRlw==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.587.0.tgz", - "integrity": "sha512-vtXTGEiw1E9Fax4LmcU2Z208gbrC8ShrdsSLmGcRPpu5NPOGBFBSDG5sy5EDNClrFxIl/Le8coQnD0EDBtx+uQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-sts": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.425.0.tgz", - "integrity": "sha512-JFojrg76oKAoBknnr9EL5N2aJ1mRCtBqXoZYST58GSx8uYdFQ89qS65VNQ8JviBXzsrCNAn4vDhZ5Ch5E6TxGQ==", - "dependencies": { - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-signing": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.425.0.tgz", - "integrity": "sha512-ZpOfgJHk7ovQ0sSwg3tU4NxFOnz53lJlkJRf7S+wxQALHM0P2MJ6LYBrZaFMVsKiJxNIdZBXD6jclgHg72ZW6Q==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/signature-v4": "^2.0.0", - "@smithy/types": "^2.3.4", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.577.0.tgz", - "integrity": "sha512-i2BPJR+rp8xmRVIGc0h1kDRFcM2J9GnClqqpc+NLSjmYadlcg4mPklisz9HzwFVcRPJ5XcGf3U4BYs5G8+iTyg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.427.0.tgz", - "integrity": "sha512-y9HxYsNvnA3KqDl8w1jHeCwz4P9CuBEtu/G+KYffLeAMBsMZmh4SIkFFCO9wE/dyYg6+yo07rYcnnIfy7WA0bw==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.425.0.tgz", - "integrity": "sha512-u7uv/iUOapIJdRgRkO3wnpYsUgV6ponsZJQgVg/8L+n+Vo5PQL5gAcIuAOwcYSKQPFaeK+KbmByI4SyOK203Vw==", - "dependencies": { - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.590.0.tgz", - "integrity": "sha512-bb8NEG2IUHqFQJsLzr1nlkTZYyokeo3bGbHwMBKZHbdF+OXrQx0kQUcaDCXYWmeydSfHXxweQEJ2U5i1YEvT/A==", - "dependencies": { - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-format-url": "3.577.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.587.0.tgz", - "integrity": "sha512-TR9+ZSjdXvXUz54ayHcCihhcvxI9W7102J1OK6MrLgBlPE7uRhAx42BR9L5lLJ86Xj3LuqPWf//o9d/zR9WVIg==", - "dependencies": { - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/token-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.427.0.tgz", - "integrity": "sha512-4E5E+4p8lJ69PBY400dJXF06LUHYx5lkKzBEsYqWWhoZcoftrvi24ltIhUDoGVLkrLcTHZIWSdFAWSos4hXqeg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/types": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.425.0.tgz", - "integrity": "sha512-6lqbmorwerN4v+J5dqbHPAsjynI0mkEF+blf+69QTaKKGaxBBVaXgqoqul9RXYcK5MMrrYRbQIMd0zYOoy90kA==", - "dependencies": { - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/util-arn-parser": { - "version": "3.568.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", - "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-endpoints": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.427.0.tgz", - "integrity": "sha512-rSyiAIFF/EVvity/+LWUqoTMJ0a25RAc9iqx0WZ4tf1UjuEXRRXxZEb+jEZg1bk+pY84gdLdx9z5E+MSJCZxNQ==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.577.0.tgz", - "integrity": "sha512-SyEGC2J+y/krFRuPgiF02FmMYhqbiIkOjDE6k4nYLJQRyS6XEAGxZoG+OHeOVEM+bsDgbxokXZiM3XKGu6qFIg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-locate-window": { - "version": "3.310.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.310.0.tgz", - "integrity": "sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.425.0.tgz", - "integrity": "sha512-22Y9iMtjGcFjGILR6/xdp1qRezlHVLyXtnpEsbuPTiernRCPk6zfAnK/ATH77r02MUjU057tdxVkd5umUBTn9Q==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - } - }, - "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.425.0.tgz", - "integrity": "sha512-SIR4F5uQeeVAi8lv4OgRirtdtNi5zeyogTuQgGi9su8F/WP1N6JqxofcwpUY5f8/oJ2UlXr/tx1f09UHfJJzvA==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/util-utf8-browser": { - "version": "3.259.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", - "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", - "dependencies": { - "tslib": "^2.3.1" - } - }, - "node_modules/@aws-sdk/xml-builder": { - "version": "3.575.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.575.0.tgz", - "integrity": "sha512-cWgAwmbFYNCFzPwxL705+lWps0F3ZvOckufd2KKoEZUmtpVw9/txUXNrPySUXSmRTSRhoatIMABNfStWR043bQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/xml-builder/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@azure/abort-controller": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", - "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", - "dependencies": { - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@azure/core-auth": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.7.2.tgz", - "integrity": "sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.1.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-auth/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-client": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", - "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-rest-pipeline": "^1.9.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/logger": "^1.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-client/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-http-compat": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz", - "integrity": "sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-client": "^1.3.0", - "@azure/core-rest-pipeline": "^1.3.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-http-compat/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-lro": { - "version": "2.5.4", - "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.4.tgz", - "integrity": "sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q==", - "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-util": "^1.2.0", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@azure/core-paging": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", - "integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==", - "dependencies": { - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.0.tgz", - "integrity": "sha512-CeuTvsXxCUmEuxH5g/aceuSl6w2EugvNHKAtKKVdiX915EjJJxAwfzNNWZreNnbxHZ2fi0zaM6wwS23x2JVqSQ==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.9.0", - "@azure/logger": "^1.0.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-tracing": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.2.tgz", - "integrity": "sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-util": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.9.0.tgz", - "integrity": "sha512-AfalUQ1ZppaKuxPPMsFEUdX6GZPB3d9paR9d/TTL7Ow2De8cJaC7ibi7kWVlFAVPCYo31OcnGymc0R89DX8Oaw==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-util/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-xml": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.2.tgz", - "integrity": "sha512-CW3MZhApe/S4iikbYKE7s83fjDBPIr2kpidX+hlGRwh7N4o1nIpQ/PfJTeioqhfqdMvRtheEl+ft64fyTaLNaA==", - "dependencies": { - "fast-xml-parser": "^4.3.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-xml/node_modules/fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/@azure/logger": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz", - "integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==", - "dependencies": { - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@azure/storage-blob": { - "version": "12.23.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.23.0.tgz", - "integrity": "sha512-c1KJ5R5hqR/HtvmFtTn/Y1BNMq45NUBp0LZH7yF8WFMET+wmESgEr0FVTu/Z5NonmfUjbgJZG5Nh8xHc5RdWGQ==", - "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-client": "^1.6.2", - "@azure/core-http-compat": "^2.0.0", - "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-rest-pipeline": "^1.10.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/core-xml": "^1.3.2", - "@azure/logger": "^1.0.0", - "events": "^3.0.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.22.13", - "chalk": "^2.4.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/code-frame/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/code-frame/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/code-frame/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.21.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.21.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.21.3", - "@babel/helper-compilation-targets": "^7.20.7", - "@babel/helper-module-transforms": "^7.21.2", - "@babel/helpers": "^7.21.0", - "@babel/parser": "^7.21.3", - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.21.3", - "@babel/types": "^7.21.3", - "convert-source-map": "^1.7.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.2", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@babel/core/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/generator": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.3.tgz", - "integrity": "sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg==", - "dev": true, - "dependencies": { - "@babel/types": "^7.23.3", - "@jridgewell/gen-mapping": "^0.3.2", - "@jridgewell/trace-mapping": "^0.3.17", - "jsesc": "^2.5.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping": { - "version": "0.3.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.20.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.20.5", - "@babel/helper-validator-option": "^7.18.6", - "browserslist": "^4.21.3", - "lru-cache": "^5.1.1", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-environment-visitor": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", - "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-function-name": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", - "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", - "dev": true, - "dependencies": { - "@babel/template": "^7.22.15", - "@babel/types": "^7.23.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-hoist-variables": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", - "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", - "dev": true, - "dependencies": { - "@babel/types": "^7.22.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.18.6", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.18.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.21.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-environment-visitor": "^7.18.9", - "@babel/helper-module-imports": "^7.18.6", - "@babel/helper-simple-access": "^7.20.2", - "@babel/helper-split-export-declaration": "^7.18.6", - "@babel/helper-validator-identifier": "^7.19.1", - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.21.2", - "@babel/types": "^7.21.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-simple-access": { - "version": "7.20.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.20.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-split-export-declaration": { - "version": "7.22.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", - "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", - "dev": true, - "dependencies": { - "@babel/types": "^7.22.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.21.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.21.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.21.0", - "@babel/types": "^7.21.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/parser": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.3.tgz", - "integrity": "sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw==", - "dev": true, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.23.2", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", - "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.0", - "@babel/helper-environment-visitor": "^7.22.20", - "@babel/helper-function-name": "^7.23.0", - "@babel/helper-hoist-variables": "^7.22.5", - "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.0", - "@babel/types": "^7.23.0", - "debug": "^4.1.0", - "globals": "^11.1.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@babel/traverse/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/types": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.3.tgz", - "integrity": "sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw==", - "dev": true, - "dependencies": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", - "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", - "dev": true, - "dependencies": { - "eslint-visitor-keys": "^3.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.1.tgz", - "integrity": "sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA==", - "dev": true, - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", - "dev": true, - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/eslintrc/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@eslint/eslintrc/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "node_modules/@eslint/eslintrc/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@eslint/eslintrc/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@eslint/eslintrc/node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", - "dev": true, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/@google-cloud/paginator": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", - "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", - "dependencies": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage": { - "version": "7.11.1", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.11.1.tgz", - "integrity": "sha512-tibLSvgw7nDohMyIelt26kBpJ59YGWA2Rzep++DFNzEzKaSuCSp56Se9iM13ZlM3j5nLzR21IBkpRN58CmvCIw==", - "dependencies": { - "@google-cloud/paginator": "^5.0.0", - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "^4.0.0", - "abort-controller": "^3.0.0", - "async-retry": "^1.3.3", - "duplexify": "^4.1.3", - "fast-xml-parser": "^4.3.0", - "gaxios": "^6.0.2", - "google-auth-library": "^9.6.3", - "html-entities": "^2.5.2", - "mime": "^3.0.0", - "p-limit": "^3.0.1", - "retry-request": "^7.0.0", - "teeny-request": "^9.0.0", - "uuid": "^8.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage/node_modules/fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/@google-cloud/storage/node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@google-cloud/storage/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", - "dev": true, - "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", - "debug": "^4.3.1", - "minimatch": "^3.0.5" - }, - "engines": { - "node": ">=10.10.0" - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true, - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@humanwhocodes/object-schema": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", - "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", - "dev": true - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { - "version": "5.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/esprima": { - "version": "4.0.1", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { - "version": "3.14.1", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/sprintf-js": { - "version": "1.0.3", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/set-array": { - "version": "1.1.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.17", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "3.1.0", - "@jridgewell/sourcemap-codec": "1.4.14" - } - }, - "node_modules/@napi-rs/snappy-android-arm-eabi": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz", - "integrity": "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-android-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz", - "integrity": "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-darwin-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz", - "integrity": "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-darwin-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz", - "integrity": "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-freebsd-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz", - "integrity": "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz", - "integrity": "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz", - "integrity": "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz", - "integrity": "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-x64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz", - "integrity": "sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-x64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.2.2.tgz", - "integrity": "sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-arm64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz", - "integrity": "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-ia32-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz", - "integrity": "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-x64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz", - "integrity": "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@project-sunbird/logger": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/@project-sunbird/logger/-/logger-0.0.9.tgz", - "integrity": "sha512-5DLFvgPBa5ffeavwY7fsXQsmZ+CM9rNC3Qmbh9pa4Ygfm10fhucijG0Aqku6WN9i5EeZXLoIOhDGr2fYjKsUqg==", - "dependencies": { - "@types/lodash": "^4.14.149", - "@types/node": "^13.9.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.15", - "winston": "^3.2.1", - "winston-daily-rotate-file": "^4.4.2" - } - }, - "node_modules/@project-sunbird/logger/node_modules/@types/node": { - "version": "13.13.52", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.52.tgz", - "integrity": "sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ==" - }, - "node_modules/@project-sunbird/logger/node_modules/async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" - }, - "node_modules/@project-sunbird/logger/node_modules/fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "node_modules/@project-sunbird/logger/node_modules/file-stream-rotator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", - "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", - "dependencies": { - "moment": "^2.29.1" - } - }, - "node_modules/@project-sunbird/logger/node_modules/logform": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", - "integrity": "sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==", - "dependencies": { - "@colors/colors": "1.6.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/@project-sunbird/logger/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/@project-sunbird/logger/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/@project-sunbird/logger/node_modules/winston": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.0.tgz", - "integrity": "sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==", - "dependencies": { - "@colors/colors": "^1.6.0", - "@dabh/diagnostics": "^2.0.2", - "async": "^3.2.3", - "is-stream": "^2.0.0", - "logform": "^2.4.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "safe-stable-stringify": "^2.3.1", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.7.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/@project-sunbird/logger/node_modules/winston-daily-rotate-file": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz", - "integrity": "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA==", - "dependencies": { - "file-stream-rotator": "^0.6.1", - "object-hash": "^2.0.1", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "winston": "^3" - } - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", - "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0" - } - }, - "node_modules/@sinonjs/samsam": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz", - "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^2.0.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - } - }, - "node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", - "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/text-encoding": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", - "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", - "dev": true - }, - "node_modules/@smithy/abort-controller": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.11.tgz", - "integrity": "sha512-MSzE1qR2JNyb7ot3blIOT3O3H0Jn06iNDEgHRaqZUwBgx5EG+VIx24Y21tlKofzYryIOcWpIohLrIIyocD6LMA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", - "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/chunked-blob-reader-native": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", - "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", - "dependencies": { - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/config-resolver": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-2.0.14.tgz", - "integrity": "sha512-K1K+FuWQoy8j/G7lAmK85o03O89s2Vvh6kMFmzEmiHUoQCRH1rzbDtMnGNiaMHeSeYJ6y79IyTusdRG+LuWwtg==", - "dependencies": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/types": "^2.3.5", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.0.tgz", - "integrity": "sha512-ygLZSSKgt9bR8HAxR9mK+U5obvAJBr6zlQuhN5soYWx/amjDoQN4dTkydTypgKe6rIbUjTILyLU+W5XFwXr4kg==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dependencies": { - "@smithy/types": "^3.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dependencies": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@smithy/credential-provider-imds": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-2.0.16.tgz", - "integrity": "sha512-tKa2xF+69TvGxJT+lnJpGrKxUuAZDLYXFhqnPEgnHz+psTpkpcB4QRjHj63+uj83KaeFJdTfW201eLZeRn6FfA==", - "dependencies": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/eventstream-codec": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-2.0.11.tgz", - "integrity": "sha512-BQCTjxhCYRZIfXapa2LmZSaH8QUBGwMZw7XRN83hrdixbLjIcj+o549zjkedFS07Ve2TlvWUI6BTzP+nv7snBA==", - "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/eventstream-serde-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.0.tgz", - "integrity": "sha512-NB7AFiPN4NxP/YCAnrvYR18z2/ZsiHiF7VtG30gshO9GbFrIb1rC8ep4NGpJSWrz6P64uhPXeo4M0UsCLnZKqw==", - "dependencies": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-browser/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.0.tgz", - "integrity": "sha512-RUQG3vQ3LX7peqqHAbmayhgrF5aTilPnazinaSGF1P0+tgM3vvIRWPHmlLIz2qFqB9LqFIxditxc8O2Z6psrRw==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-config-resolver/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.0.tgz", - "integrity": "sha512-baRPdMBDMBExZXIUAoPGm/hntixjt/VFpU6+VmCyiYJYzRHRxoaI1MN+5XE+hIS8AJ2GCHLMFEIOLzq9xx1EgQ==", - "dependencies": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-node/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.0.tgz", - "integrity": "sha512-HNFfShmotWGeAoW4ujP8meV9BZavcpmerDbPIjkJbxKbN8RsUcpRQ/2OyIxWNxXNH2GWCAxuSB7ynmIGJlQ3Dw==", - "dependencies": { - "@smithy/eventstream-codec": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/eventstream-codec": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.0.0.tgz", - "integrity": "sha512-PUtyEA0Oik50SaEFCZ0WPVtF9tz/teze2fDptW6WRXl+RrEenH8UbEjudOz8iakiMl3lE3lCVqYf2Y+znL8QFQ==", - "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/fetch-http-handler": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.2.tgz", - "integrity": "sha512-K7aRtRuaBjzlk+jWWeyfDTLAmRRvmA4fU8eHUXtjsuEDgi3f356ZE32VD2ssxIH13RCLVZbXMt5h7wHzYiSuVA==", - "dependencies": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/hash-blob-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.0.0.tgz", - "integrity": "sha512-/Wbpdg+bwJvW7lxR/zpWAc1/x/YkcqguuF2bAzkJrvXriZu1vm8r+PUdE4syiVwQg7PPR2dXpi3CLBb9qRDaVQ==", - "dependencies": { - "@smithy/chunked-blob-reader": "^3.0.0", - "@smithy/chunked-blob-reader-native": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/hash-blob-browser/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-node": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-2.0.11.tgz", - "integrity": "sha512-PbleVugN2tbhl1ZoNWVrZ1oTFFas/Hq+s6zGO8B9bv4w/StTriTKA9W+xZJACOj9X7zwfoTLbscM+avCB1KqOQ==", - "dependencies": { - "@smithy/types": "^2.3.5", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/hash-stream-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.0.0.tgz", - "integrity": "sha512-J0i7de+EgXDEGITD4fxzmMX8CyCNETTIRXlxjMiNUvvu76Xn3GJ31wQR85ynlPk2wI1lqoknAFJaD1fiNDlbIA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/invalid-dependency": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-2.0.11.tgz", - "integrity": "sha512-zazq99ujxYv/NOf9zh7xXbNgzoVLsqE0wle8P/1zU/XdhPi/0zohTPKWUzIxjGdqb5hkkwfBkNkl5H+LE0mvgw==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/is-array-buffer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.0.0.tgz", - "integrity": "sha512-z3PjFjMyZNI98JFRJi/U0nGoLWMSJlDjAW4QUX2WNZLas5C0CmVV6LJ01JI0k90l7FvpmixjWxPFmENSClQ7ug==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/md5-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.0.tgz", - "integrity": "sha512-Tm0vrrVzjlD+6RCQTx7D3Ls58S3FUH1ZCtU1MIh/qQmaOo1H9lMN2as6CikcEwgattnA9SURSdoJJ27xMcEfMA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/middleware-content-length": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-2.0.13.tgz", - "integrity": "sha512-Md2kxWpaec3bXp1oERFPQPBhOXCkGSAF7uc1E+4rkwjgw3/tqAXRtbjbggu67HJdwaif76As8AV6XxbD1HzqTQ==", - "dependencies": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-endpoint": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-2.0.11.tgz", - "integrity": "sha512-mCugsvB15up6fqpzUEpMT4CuJmFkEI+KcozA7QMzYguXCaIilyMKsyxgamwmr+o7lo3QdjN0//XLQ9bWFL129g==", - "dependencies": { - "@smithy/middleware-serde": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-retry": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-2.0.16.tgz", - "integrity": "sha512-Br5+0yoiMS0ugiOAfJxregzMMGIRCbX4PYo1kDHtLgvkA/d++aHbnHB819m5zOIAMPvPE7AThZgcsoK+WOsUTA==", - "dependencies": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/protocol-http": "^3.0.7", - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-retry": "^2.0.4", - "tslib": "^2.5.0", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-retry/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@smithy/middleware-serde": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-2.0.11.tgz", - "integrity": "sha512-NuxnjMyf4zQqhwwdh0OTj5RqpnuT6HcH5Xg5GrPijPcKzc2REXVEVK4Yyk8ckj8ez1XSj/bCmJ+oNjmqB02GWA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-stack": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-2.0.5.tgz", - "integrity": "sha512-bVQU/rZzBY7CbSxIrDTGZYnBWKtIw+PL/cRc9B7etZk1IKSOe0NvKMJyWllfhfhrTeMF6eleCzOihIQympAvPw==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-config-provider": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-2.1.1.tgz", - "integrity": "sha512-1lF6s1YWBi1LBu2O30tD3jyTgMtuvk/Z1twzXM4GPYe4dmZix4nNREPJIPOcfFikNU2o0eTYP80+izx5F2jIJA==", - "dependencies": { - "@smithy/property-provider": "^2.0.12", - "@smithy/shared-ini-file-loader": "^2.2.0", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-http-handler": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.1.7.tgz", - "integrity": "sha512-PQIKZXlp3awCDn/xNlCSTFE7aYG/5Tx33M05NfQmWYeB5yV1GZZOSz4dXpwiNJYTXb9jPqjl+ueXXkwtEluFFA==", - "dependencies": { - "@smithy/abort-controller": "^2.0.11", - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/property-provider": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-2.0.12.tgz", - "integrity": "sha512-Un/OvvuQ1Kg8WYtoMCicfsFFuHb/TKL3pCA6ZIo/WvNTJTR94RtoRnL7mY4XkkUAoFMyf6KjcQJ76y1FX7S5rw==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/protocol-http": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-3.0.7.tgz", - "integrity": "sha512-HnZW8y+r66ntYueCDbLqKwWcMNWW8o3eVpSrHNluwtBJ/EUWfQHRKSiu6vZZtc6PGfPQWgVfucoCE/C3QufMAA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/querystring-builder": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-2.0.11.tgz", - "integrity": "sha512-b4kEbVMxpmfv2VWUITn2otckTi7GlMteZQxi+jlwedoATOGEyrCJPfRcYQJjbCi3fZ2QTfh3PcORvB27+j38Yg==", - "dependencies": { - "@smithy/types": "^2.3.5", - "@smithy/util-uri-escape": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/querystring-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.11.tgz", - "integrity": "sha512-YXe7jhi7s3dQ0Fu9dLoY/gLu6NCyy8tBWJL/v2c9i7/RLpHgKT+uT96/OqZkHizCJ4kr0ZD46tzMjql/o60KLg==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/service-error-classification": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-2.0.4.tgz", - "integrity": "sha512-77506l12I5gxTZqBkx3Wb0RqMG81bMYLaVQ+EqIWFwQDJRs5UFeXogKxSKojCmz1wLUziHZQXm03MBzPQiumQw==", - "dependencies": { - "@smithy/types": "^2.3.5" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/shared-ini-file-loader": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-2.2.0.tgz", - "integrity": "sha512-xFXqs4vAb5BdkzHSRrTapFoaqS4/3m/CGZzdw46fBjYZ0paYuLAoMY60ICCn1FfGirG+PiJ3eWcqJNe4/SkfyA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/signature-v4": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-2.0.11.tgz", - "integrity": "sha512-EFVU1dT+2s8xi227l1A9O27edT/GNKvyAK6lZnIZ0zhIHq/jSLznvkk15aonGAM1kmhmZBVGpI7Tt0odueZK9A==", - "dependencies": { - "@smithy/eventstream-codec": "^2.0.11", - "@smithy/is-array-buffer": "^2.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-uri-escape": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/smithy-client": { - "version": "2.1.10", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-2.1.10.tgz", - "integrity": "sha512-2OEmZDiW1Z196QHuQZ5M6cBE8FCSG0H2HADP1G+DY8P3agsvb0YJyfhyKuJbxIQy15tr3eDAK6FOrlbxgKOOew==", - "dependencies": { - "@smithy/middleware-stack": "^2.0.5", - "@smithy/types": "^2.3.5", - "@smithy/util-stream": "^2.0.15", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/types": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.3.5.tgz", - "integrity": "sha512-ehyDt8M9hehyxrLQGoA1BGPou8Js1Ocoh5M0ngDhJMqbFmNK5N6Xhr9/ZExWkyIW8XcGkiMPq3ZUEE0ScrhbuQ==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/url-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.11.tgz", - "integrity": "sha512-h89yXMCCF+S5k9XIoKltMIWTYj+FcEkU/IIFZ6RtE222fskOTL4Iak6ZRG+ehSvZDt8yKEcxqheTDq7JvvtK3g==", - "dependencies": { - "@smithy/querystring-parser": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/util-base64": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-2.0.0.tgz", - "integrity": "sha512-Zb1E4xx+m5Lud8bbeYi5FkcMJMnn+1WUnJF3qD7rAdXpaL7UjkFQLdmW5fHadoKbdHpwH9vSR8EyTJFHJs++tA==", - "dependencies": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-body-length-browser": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz", - "integrity": "sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg==", - "dependencies": { - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/util-body-length-node": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-2.1.0.tgz", - "integrity": "sha512-/li0/kj/y3fQ3vyzn36NTLGmUwAICb7Jbe/CsWCktW363gh1MOcpEcSO3mJ344Gv2dqz8YJCLQpb6hju/0qOWw==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-buffer-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.0.0.tgz", - "integrity": "sha512-/YNnLoHsR+4W4Vf2wL5lGv0ksg8Bmk3GEGxn2vEQt52AQaPSCuaO5PM5VM7lP1K9qHRKHwrPGktqVoAHKWHxzw==", - "dependencies": { - "@smithy/is-array-buffer": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-config-provider": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-2.0.0.tgz", - "integrity": "sha512-xCQ6UapcIWKxXHEU4Mcs2s7LcFQRiU3XEluM2WcCjjBtQkUN71Tb+ydGmJFPxMUrW/GWMgQEEGipLym4XG0jZg==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-browser": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-2.0.14.tgz", - "integrity": "sha512-NupG7SWUucm3vJrvlpt9jG1XeoPJphjcivgcUUXhDJbUPy4F04LhlTiAhWSzwlCNcF8OJsMvZ/DWbpYD3pselw==", - "dependencies": { - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-node": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-2.0.18.tgz", - "integrity": "sha512-+3jMom/b/Cdp21tDnY4vKu249Al+G/P0HbRbct7/aSZDlROzv1tksaYukon6UUv7uoHn+/McqnsvqZHLlqvQ0g==", - "dependencies": { - "@smithy/config-resolver": "^2.0.14", - "@smithy/credential-provider-imds": "^2.0.16", - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@smithy/util-endpoints": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.1.tgz", - "integrity": "sha512-ZRT0VCOnKlVohfoABMc8lWeQo/JEFuPWctfNRXgTHbyOVssMOLYFUNWukxxiHRGVAhV+n3c0kPW+zUqckjVPEA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-hex-encoding": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz", - "integrity": "sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-middleware": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-2.0.4.tgz", - "integrity": "sha512-Pbu6P4MBwRcjrLgdTR1O4Y3c0sTZn2JdOiJNcgL7EcIStcQodj+6ZTXtbyU/WTEU3MV2NMA10LxFc3AWHZ3+4A==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-retry": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-2.0.4.tgz", - "integrity": "sha512-b+n1jBBKc77C1E/zfBe1Zo7S9OXGBiGn55N0apfhZHxPUP/fMH5AhFUUcWaJh7NAnah284M5lGkBKuhnr3yK5w==", - "dependencies": { - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@smithy/util-stream": { - "version": "2.0.15", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-2.0.15.tgz", - "integrity": "sha512-A/hkYJPH2N5MCWYvky4tTpQihpYAEzqnUfxDyG3L/yMndy/2sLvxnyQal9Opuj1e9FiKSTeMyjnU9xxZGs0mRw==", - "dependencies": { - "@smithy/fetch-http-handler": "^2.2.2", - "@smithy/node-http-handler": "^2.1.7", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-uri-escape": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.0.0.tgz", - "integrity": "sha512-ebkxsqinSdEooQduuk9CbKcI+wheijxEb3utGXkCoYQkJnwTnLbH1JXGimJtUkQwNQbsbuYwG2+aFVyZf5TLaw==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-utf8": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.0.0.tgz", - "integrity": "sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ==", - "dependencies": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-waiter": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.0.0.tgz", - "integrity": "sha512-+fEXJxGDLCoqRKVSmo0auGxaqbiCo+8oph+4auefYjaNxjOLKSY2MxVQfRzo65PaZv4fr+5lWg+au7vSuJJ/zw==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-waiter/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-waiter/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tsconfig/node10": { - "version": "1.0.9", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.3", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" - }, - "node_modules/@types/chai": { - "version": "4.3.4", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/chai-as-promised": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.6.tgz", - "integrity": "sha512-cQLhk8fFarRVZAXUQV1xEnZgMoPxqKojBvRkqPCKPQCzEhpbbSKl1Uu75kDng7k5Ln6LQLUmNBjLlFthCgm1NA==", - "dev": true, - "dependencies": { - "@types/chai": "*" - } - }, - "node_modules/@types/chai-spies": { - "version": "1.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/chai": "*" - } - }, - "node_modules/@types/compression": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.3.tgz", - "integrity": "sha512-rKquEGjebqizyHNMOpaE/4FdYR5VQiWFeesqYfvJU0seSEyB4625UGhNOO/qIkH10S3wftiV7oefc8WdLZ/gCQ==", - "dev": true, - "dependencies": { - "@types/express": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.35", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/cookiejar": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/debug": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", - "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", - "dependencies": { - "@types/ms": "*" - } - }, - "node_modules/@types/express": { - "version": "4.17.17", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.33", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/http-errors": { - "version": "2.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/kafkajs": { - "version": "1.9.0", - "dev": true, - "license": "MIT", - "dependencies": { - "kafkajs": "*" - } - }, - "node_modules/@types/knex": { - "version": "0.16.1", - "dev": true, - "license": "MIT", - "dependencies": { - "knex": "*" - } - }, - "node_modules/@types/lodash": { - "version": "4.14.191", - "license": "MIT" - }, - "node_modules/@types/mime": { - "version": "3.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mocha": { - "version": "10.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mock-knex": { - "version": "0.4.6", - "resolved": "https://registry.npmjs.org/@types/mock-knex/-/mock-knex-0.4.6.tgz", - "integrity": "sha512-7MHM9v9ZgFpAZh2NfCyYeEJKlYJSf/mVRAQ324AJZGB9fbo8dVa8u9nxibyJzwwMXYlbRXLNbKEfoK2xBJFgUw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/moment": { - "version": "2.13.0", - "dev": true, - "license": "MIT", - "dependencies": { - "moment": "*" - } - }, - "node_modules/@types/ms": { - "version": "0.7.34", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", - "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" - }, - "node_modules/@types/node": { - "version": "18.15.3", - "license": "MIT" - }, - "node_modules/@types/pegjs": { - "version": "0.10.6", - "resolved": "https://registry.npmjs.org/@types/pegjs/-/pegjs-0.10.6.tgz", - "integrity": "sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==" - }, - "node_modules/@types/pg": { - "version": "8.6.6", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/qs": { - "version": "6.9.7", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - } - }, - "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "node_modules/@types/sinon": { - "version": "17.0.3", - "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.3.tgz", - "integrity": "sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==", - "dev": true, - "dependencies": { - "@types/sinonjs__fake-timers": "*" - } - }, - "node_modules/@types/sinonjs__fake-timers": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", - "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", - "dev": true - }, - "node_modules/@types/slug": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/@types/slug/-/slug-5.0.8.tgz", - "integrity": "sha512-mblTWR1OST257k1gZ3QvqG+ERSr8Ea6dyM1FH6Jtm4jeXi0/r0/95VNctofuiywPxCVQuE8AuFoqmvJ4iVUlXQ==", - "dev": true - }, - "node_modules/@types/superagent": { - "version": "3.8.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/cookiejar": "*", - "@types/node": "*" - } - }, - "node_modules/@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" - }, - "node_modules/@types/triple-beam": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.3.tgz", - "integrity": "sha512-6tOUG+nVHn0cJbVp25JFayS5UE6+xlbcNF9Lo9mU7U0zk3zeUShZied4YEQZjy1JBF043FSkdXw8YkUJuVtB5g==" - }, - "node_modules/@types/uuid": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz", - "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ==", - "dev": true - }, - "node_modules/@types/validator": { - "version": "13.11.10", - "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.11.10.tgz", - "integrity": "sha512-e2PNXoXLr6Z+dbfx5zSh9TRlXJrELycxiaXznp4S5+D2M3b9bqJEitNHA5923jhnB2zzFiZHa2f0SI1HoIahpg==" - }, - "node_modules/@typescript-eslint/eslint-plugin": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.12.0.tgz", - "integrity": "sha512-7F91fcbuDf/d3S8o21+r3ZncGIke/+eWk0EpO21LXhDfLahriZF9CGj4fbAetEjlaBdjdSm9a6VeXbpbT6Z40Q==", - "dev": true, - "dependencies": { - "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/type-utils": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "graphemer": "^1.4.0", - "ignore": "^5.3.1", - "natural-compare": "^1.4.0", - "ts-api-utils": "^1.3.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^7.0.0", - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.12.0.tgz", - "integrity": "sha512-dm/J2UDY3oV3TKius2OUZIFHsomQmpHtsV0FTh1WO8EKgHLQ1QCADUqscPgTpU+ih1e21FQSRjXckHn3txn6kQ==", - "dev": true, - "dependencies": { - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.12.0.tgz", - "integrity": "sha512-itF1pTnN6F3unPak+kutH9raIkL3lhH1YRPGgt7QQOh43DQKVJXmWkpb+vpc/TiDHs6RSd9CTbDsc/Y+Ygq7kg==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/type-utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.12.0.tgz", - "integrity": "sha512-lib96tyRtMhLxwauDWUp/uW3FMhLA6D0rJ8T7HmH7x23Gk1Gwwu8UZ94NMXBvOELn6flSPiBrCKlehkiXyaqwA==", - "dev": true, - "dependencies": { - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "debug": "^4.3.4", - "ts-api-utils": "^1.3.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@typescript-eslint/types": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.12.0.tgz", - "integrity": "sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg==", - "dev": true, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.12.0.tgz", - "integrity": "sha512-5bwqLsWBULv1h6pn7cMW5dXX/Y2amRqLaKqsASVwbBHMZSnHqE/HN4vT4fE0aFsiwxYvr98kqOWh1a8ZKXalCQ==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4", - "globby": "^11.1.0", - "is-glob": "^4.0.3", - "minimatch": "^9.0.4", - "semver": "^7.6.0", - "ts-api-utils": "^1.3.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dev": true, - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@typescript-eslint/utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.12.0.tgz", - "integrity": "sha512-Y6hhwxwDx41HNpjuYswYp6gDbkiZ8Hin9Bf5aJQn1bpTs3afYY4GX+MPYxma8jtoIV2GRwTM/UJm/2uGCVv+DQ==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.12.0.tgz", - "integrity": "sha512-uZk7DevrQLL3vSnfFl5bj4sL75qC9D6EdjemIdbtkuUmIheWpuiiylSY01JxJE7+zGrOWDZrp1WxOuDntvKrHQ==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.12.0", - "eslint-visitor-keys": "^3.4.3" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true - }, - "node_modules/abbrev": { - "version": "1.1.1", - "dev": true, - "license": "ISC" - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/accepts": { - "version": "1.3.8", - "license": "MIT", - "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", - "dev": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/agent-base/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/agent-base/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/aggregate-error": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ajv": { - "version": "8.12.0", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ansi-colors": { - "version": "4.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/append-transform": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "default-require-extensions": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, - "node_modules/archy": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/are-we-there-yet": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", - "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", - "optional": true, - "dependencies": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "node_modules/arg": { - "version": "4.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/argparse": { - "version": "2.0.1", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "license": "MIT" - }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "engines": { - "node": ">=8" - } - }, - "node_modules/assertion-error": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/async": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", - "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", - "dependencies": { - "lodash": "^4.17.14" - } - }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "dependencies": { - "retry": "0.13.1" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "license": "MIT" - }, - "node_modules/available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/aws-sdk": { - "version": "2.1472.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1472.0.tgz", - "integrity": "sha512-U7kAHRbvTy753IXKV8Oom/AqlqnsbXG+Kw5gRbKi6VcsZ3hR/EpNMzdRXTWO5U415bnLWGo8WAqIz67PIaaKsw==", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.5.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/aws-sdk/node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "node_modules/aws-sdk/node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/aws-sdk/node_modules/uuid": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", - "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/axios": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz", - "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==", - "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/big-integer": { - "version": "1.6.52", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", - "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/bignumber.js": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", - "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", - "engines": { - "node": "*" - } - }, - "node_modules/binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "dependencies": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/binary-extensions": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, - "node_modules/bintrees": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", - "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" - }, - "node_modules/bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/bowser": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", - "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" - }, - "node_modules/brace-expansion": { - "version": "1.1.11", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browser-stdout": { - "version": "1.3.1", - "dev": true, - "license": "ISC" - }, - "node_modules/browserslist": { - "version": "4.21.5", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - } - ], - "license": "MIT", - "dependencies": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "node_modules/buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "optional": true, - "dependencies": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "node_modules/buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", - "optional": true - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" - }, - "node_modules/buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", - "optional": true - }, - "node_modules/buffermaker": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz", - "integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==", - "dependencies": { - "long": "1.1.2" - } - }, - "node_modules/buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", - "engines": { - "node": ">=0.2.0" - } - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/caching-transform": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "hasha": "^5.0.0", - "make-dir": "^3.0.0", - "package-hash": "^4.0.0", - "write-file-atomic": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/call-bind": { - "version": "1.0.2", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "6.3.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001466", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chai": { - "version": "4.3.7", - "dev": true, - "license": "MIT", - "dependencies": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.2", - "deep-eql": "^4.1.2", - "get-func-name": "^2.0.0", - "loupe": "^2.3.1", - "pathval": "^1.1.1", - "type-detect": "^4.0.5" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/chai-as-promised": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", - "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", - "dev": true, - "dependencies": { - "check-error": "^1.0.2" - }, - "peerDependencies": { - "chai": ">= 2.1.2 < 5" - } - }, - "node_modules/chai-http": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/chai": "4", - "@types/superagent": "^3.8.3", - "cookiejar": "^2.1.1", - "is-ip": "^2.0.0", - "methods": "^1.1.2", - "qs": "^6.5.1", - "superagent": "^3.7.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/chai-spies": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4.0.0" - }, - "peerDependencies": { - "chai": "*" - } - }, - "node_modules/chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "dependencies": { - "traverse": ">=0.3.0 <0.4" - }, - "engines": { - "node": "*" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/check-error": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/chokidar": { - "version": "3.5.3", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "license": "MIT", - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "optional": true - }, - "node_modules/clean-stack": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/cliui": { - "version": "7.0.4", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/color": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", - "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", - "dependencies": { - "color-convert": "^1.9.3", - "color-string": "^1.6.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "license": "MIT" - }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/color/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "node_modules/colorette": { - "version": "2.0.19", - "license": "MIT" - }, - "node_modules/colors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", - "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", - "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", - "dependencies": { - "color": "^3.1.3", - "text-hex": "1.0.x" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "9.5.0", - "license": "MIT", - "engines": { - "node": "^12.20.0 || >=14" - } - }, - "node_modules/commondir": { - "version": "1.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/component-emitter": { - "version": "1.3.0", - "dev": true, - "license": "MIT" - }, - "node_modules/compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "dependencies": { - "mime-db": ">= 1.43.0 < 2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/compression/node_modules/bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/compression/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "license": "MIT" - }, - "node_modules/console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "optional": true - }, - "node_modules/content-disposition": { - "version": "0.5.4", - "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/convert-source-map": { - "version": "1.9.0", - "dev": true, - "license": "MIT" - }, - "node_modules/cookie": { - "version": "0.5.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.6", - "license": "MIT" - }, - "node_modules/cookiejar": { - "version": "2.1.4", - "dev": true, - "license": "MIT" - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "license": "MIT" - }, - "node_modules/create-require": { - "version": "1.1.1", - "dev": true, - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/cycle": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", - "integrity": "sha512-TVF6svNzeQCOpjCqsy0/CSy8VgObG3wXusJ73xW2GbG5rGx7lC8zxDSURicsXI2UsGdi2L0QNRCi745/wUDvsA==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/dateformat": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.0.0.tgz", - "integrity": "sha512-k6+FtJ8RoNx9V0yHo6lURQWlFqc8wbo0t/kocHvfMJQiXTW+izR6bubmB9HeWuUiZFtpMrAm+wPMGmGhfPbNlQ==", - "engines": { - "node": "*" - } - }, - "node_modules/debug": { - "version": "2.6.9", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/decamelize": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", - "optional": true, - "dependencies": { - "mimic-response": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/deep-eql": { - "version": "4.1.3", - "dev": true, - "license": "MIT", - "dependencies": { - "type-detect": "^4.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "optional": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, - "node_modules/default-require-extensions": { - "version": "3.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "strip-bom": "^4.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "optional": true - }, - "node_modules/denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/depd": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", - "optional": true, - "bin": { - "detect-libc": "bin/detect-libc.js" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/diff": { - "version": "5.0.0", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/diff-json": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash.difference": ">= 4.0.0", - "lodash.find": ">= 4.0.0", - "lodash.intersection": ">= 4.0.0", - "lodash.keyby": ">= 4.0.0" - } - }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "dependencies": { - "esutils": "^2.0.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/dottie": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz", - "integrity": "sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==" - }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, - "node_modules/duplexify/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "license": "MIT" - }, - "node_modules/electron-to-chromium": { - "version": "1.4.330", - "dev": true, - "license": "ISC" - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "devOptional": true, - "license": "MIT" - }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/encodeurl": { - "version": "1.0.2", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/es6-error": { - "version": "4.1.1", - "dev": true, - "license": "MIT" - }, - "node_modules/escalade": { - "version": "3.1.1", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "license": "MIT" - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", - "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "@ungap/structured-clone": "^1.2.0", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.3.2", - "doctrine": "^3.0.0", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.2", - "eslint-visitor-keys": "^3.4.3", - "espree": "^9.6.1", - "esquery": "^1.4.2", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "globals": "^13.19.0", - "graphemer": "^1.4.0", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-yaml": "^4.1.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3", - "strip-ansi": "^6.0.1", - "text-table": "^0.2.0" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", - "dev": true, - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/eslint/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/eslint/node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/eslint/node_modules/globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint/node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "node_modules/eslint/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/eslint/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/eslint/node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/esm": { - "version": "3.2.25", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/espree": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", - "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", - "dev": true, - "dependencies": { - "acorn": "^8.9.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.4.1" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", - "dev": true, - "dependencies": { - "estraverse": "^5.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/etag": { - "version": "1.8.1", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "engines": { - "node": ">=6" - } - }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "engines": { - "node": ">=0.8.x" - } - }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "optional": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/express": { - "version": "4.18.2", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.1", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.5.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.2.0", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.7", - "qs": "6.11.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - } - }, - "node_modules/express/node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/express/node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "license": "MIT" - }, - "node_modules/eyes": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", - "integrity": "sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==", - "engines": { - "node": "> 0.1.90" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "license": "MIT" - }, - "node_modules/fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true - }, - "node_modules/fast-safe-stringify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", - "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" - }, - "node_modules/fast-xml-parser": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", - "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", - "funding": [ - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - }, - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", - "dev": true, - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "dependencies": { - "pend": "~1.2.0" - } - }, - "node_modules/fecha": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", - "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" - }, - "node_modules/file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", - "dev": true, - "dependencies": { - "flat-cache": "^3.0.4" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "node_modules/file-stream-rotator": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.2.1.tgz", - "integrity": "sha512-Qf6xva6g0fWqI3LR48eOa8ubF+AP6ftLUSt9Uin6XadP5s3d4vD0J52l/JjmMPiFleCtsVJJ6cyMU2GNRLXQIQ==", - "dependencies": { - "moment": "^2.11.2" - } - }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/finalhandler": { - "version": "1.2.0", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/find-cache-dir": { - "version": "3.3.2", - "dev": true, - "license": "MIT", - "dependencies": { - "commondir": "^1.0.1", - "make-dir": "^3.0.2", - "pkg-dir": "^4.1.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/avajs/find-cache-dir?sponsor=1" - } - }, - "node_modules/find-up": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/flat": { - "version": "5.0.2", - "dev": true, - "license": "BSD-3-Clause", - "bin": { - "flat": "cli.js" - } - }, - "node_modules/flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", - "dev": true, - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", - "dev": true - }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, - "node_modules/follow-redirects": { - "version": "1.15.2", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dependencies": { - "is-callable": "^1.1.3" - } - }, - "node_modules/foreground-child": { - "version": "2.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/form-data": { - "version": "4.0.0", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/formidable": { - "version": "1.2.6", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://ko-fi.com/tunnckoCore/commissions" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "0.5.2", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fromentries": { - "version": "1.3.2", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "optional": true - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/fsevents": { - "version": "2.3.2", - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.1", - "license": "MIT" - }, - "node_modules/gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==", - "optional": true, - "dependencies": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "node_modules/gauge/node_modules/ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gauge/node_modules/is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", - "optional": true, - "dependencies": { - "number-is-nan": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gauge/node_modules/string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", - "optional": true, - "dependencies": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gauge/node_modules/strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "optional": true, - "dependencies": { - "ansi-regex": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gaxios": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.6.0.tgz", - "integrity": "sha512-bpOZVQV5gthH/jVCSuYuokRo2bTKOcuBiVWpjmTn6C5Agl5zclGfTljuGsQZxwwDBkli+YhZhP4TdlqTnhOezQ==", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gaxios/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", - "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", - "dependencies": { - "gaxios": "^6.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true, - "engines": { - "node": "*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.2.0", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/getopts": { - "version": "2.3.0", - "license": "MIT" - }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "optional": true - }, - "node_modules/glob": { - "version": "7.2.0", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/globals": { - "version": "11.12.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/google-auth-library": { - "version": "9.10.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.10.0.tgz", - "integrity": "sha512-ol+oSa5NbcGdDqA+gZ3G3mev59OHBZksBTxY/tYwjtcp1H/scAFwJfSQU9/1RALoyZ7FslNbke8j4i3ipwlyuQ==", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.10", - "dev": true, - "license": "ISC" - }, - "node_modules/graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true - }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/has": { - "version": "1.0.3", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.0.3", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dependencies": { - "has-symbols": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "optional": true - }, - "node_modules/hasha": { - "version": "5.2.2", - "dev": true, - "license": "MIT", - "dependencies": { - "is-stream": "^2.0.0", - "type-fest": "^0.8.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/he": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "bin": { - "he": "bin/he" - } - }, - "node_modules/html-entities": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", - "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/mdevils" - }, - { - "type": "patreon", - "url": "https://patreon.com/mdevils" - } - ] - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "dev": true, - "license": "MIT" - }, - "node_modules/http-errors": { - "version": "2.0.0", - "license": "MIT", - "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/http-proxy-agent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/http-status": { - "version": "1.6.2", - "license": "BSD-3-Clause", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", - "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, - "node_modules/ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", - "dev": true, - "engines": { - "node": ">= 4" - } - }, - "node_modules/ignore-by-default": { - "version": "1.0.1", - "dev": true, - "license": "ISC" - }, - "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/import-fresh/node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/indent-string": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/inflection": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", - "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==", - "engines": [ - "node >= 0.4.0" - ] - }, - "node_modules/inflight": { - "version": "1.0.6", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "license": "ISC" - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "optional": true - }, - "node_modules/interpret": { - "version": "2.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/ip-regex": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-core-module": { - "version": "2.11.0", - "license": "MIT", - "dependencies": { - "has": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-ip": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ip-regex": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-plain-obj": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "dependencies": { - "which-typed-array": "^1.1.11" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/is-unicode-supported": { - "version": "0.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-windows": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "license": "MIT" - }, - "node_modules/isexe": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.0", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-hook": { - "version": "3.0.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "append-transform": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "4.0.3", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.7.5", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.0.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-processinfo": { - "version": "2.0.3", - "dev": true, - "license": "ISC", - "dependencies": { - "archy": "^1.0.0", - "cross-spawn": "^7.0.3", - "istanbul-lib-coverage": "^3.2.0", - "p-map": "^3.0.0", - "rimraf": "^3.0.0", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-processinfo/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^3.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-report/node_modules/supports-color": { - "version": "7.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/istanbul-reports": { - "version": "3.1.5", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true - }, - "node_modules/js-yaml": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "2.5.2", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "license": "MIT" - }, - "node_modules/json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true - }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "license": "ISC" - }, - "node_modules/json5": { - "version": "2.2.3", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/just-extend": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", - "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", - "dev": true - }, - "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "dependencies": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "dependencies": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/kafka-node": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-5.0.0.tgz", - "integrity": "sha512-dD2ga5gLcQhsq1yNoQdy1MU4x4z7YnXM5bcG9SdQuiNr5KKuAmXixH1Mggwdah5o7EfholFbcNDPSVA6BIfaug==", - "dependencies": { - "async": "^2.6.2", - "binary": "~0.3.0", - "bl": "^2.2.0", - "buffer-crc32": "~0.2.5", - "buffermaker": "~1.2.0", - "debug": "^2.1.3", - "denque": "^1.3.0", - "lodash": "^4.17.4", - "minimatch": "^3.0.2", - "nested-error-stacks": "^2.0.0", - "optional": "^0.1.3", - "retry": "^0.10.1", - "uuid": "^3.0.0" - }, - "engines": { - "node": ">=8.5.1" - }, - "optionalDependencies": { - "snappy": "^6.0.1" - } - }, - "node_modules/kafka-node/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/kafka-node/node_modules/retry": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", - "integrity": "sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==", - "engines": { - "node": "*" - } - }, - "node_modules/kafkajs": { - "version": "2.2.4", - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/kafkajs-snappy": { - "version": "1.1.0", - "license": "MIT", - "dependencies": { - "snappyjs": "^0.6.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/kafkajs-snappy-typescript": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/kafkajs-snappy-typescript/-/kafkajs-snappy-typescript-1.0.3.tgz", - "integrity": "sha512-H5CiRKQ+RGJprWixvY5gDF3ofZEqQA0FFo0ePNgDCoJuH87n3gGUogtlUuei6A8c1ad/5qsQS3f9SSnmB/me4A==", - "dependencies": { - "snappy": "^7.2.2" - } - }, - "node_modules/kafkajs-snappy-typescript/node_modules/snappy": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.2.2.tgz", - "integrity": "sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==", - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "optionalDependencies": { - "@napi-rs/snappy-android-arm-eabi": "7.2.2", - "@napi-rs/snappy-android-arm64": "7.2.2", - "@napi-rs/snappy-darwin-arm64": "7.2.2", - "@napi-rs/snappy-darwin-x64": "7.2.2", - "@napi-rs/snappy-freebsd-x64": "7.2.2", - "@napi-rs/snappy-linux-arm-gnueabihf": "7.2.2", - "@napi-rs/snappy-linux-arm64-gnu": "7.2.2", - "@napi-rs/snappy-linux-arm64-musl": "7.2.2", - "@napi-rs/snappy-linux-x64-gnu": "7.2.2", - "@napi-rs/snappy-linux-x64-musl": "7.2.2", - "@napi-rs/snappy-win32-arm64-msvc": "7.2.2", - "@napi-rs/snappy-win32-ia32-msvc": "7.2.2", - "@napi-rs/snappy-win32-x64-msvc": "7.2.2" - } - }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "dependencies": { - "json-buffer": "3.0.1" - } - }, - "node_modules/knex": { - "version": "2.4.2", - "license": "MIT", - "dependencies": { - "colorette": "2.0.19", - "commander": "^9.1.0", - "debug": "4.3.4", - "escalade": "^3.1.1", - "esm": "^3.2.25", - "get-package-type": "^0.1.0", - "getopts": "2.3.0", - "interpret": "^2.2.0", - "lodash": "^4.17.21", - "pg-connection-string": "2.5.0", - "rechoir": "^0.8.0", - "resolve-from": "^5.0.0", - "tarn": "^3.0.2", - "tildify": "2.0.0" - }, - "bin": { - "knex": "bin/cli.js" - }, - "engines": { - "node": ">=12" - }, - "peerDependenciesMeta": { - "better-sqlite3": { - "optional": true - }, - "mysql": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-native": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, - "node_modules/knex/node_modules/debug": { - "version": "4.3.4", - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/knex/node_modules/ms": { - "version": "2.1.2", - "license": "MIT" - }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/locate-path": { - "version": "6.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash": { - "version": "4.17.21", - "license": "MIT" - }, - "node_modules/lodash.difference": { - "version": "4.5.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.find": { - "version": "4.6.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.flattendeep": { - "version": "4.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", - "dev": true - }, - "node_modules/lodash.intersection": { - "version": "4.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.keyby": { - "version": "4.6.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true - }, - "node_modules/log-symbols": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/logform": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz", - "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^2.3.3", - "ms": "^2.1.1", - "triple-beam": "^1.2.0" - } - }, - "node_modules/logform/node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/logform/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/long": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz", - "integrity": "sha512-pjR3OP1X2VVQhCQlrq3s8UxugQsuoucwMOn9Yj/kN/61HMc+lDFJS5bvpNEHneZ9NVaSm8gNWxZvtGS7lqHb3Q==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/loupe": { - "version": "2.3.6", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.0" - } - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^6.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "dev": true, - "license": "ISC" - }, - "node_modules/media-typer": { - "version": "0.3.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.1", - "license": "MIT" - }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/methods": { - "version": "1.1.2", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", - "dev": true, - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "optional": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/minimatch": { - "version": "5.0.1", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/minimatch/node_modules/brace-expansion": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "devOptional": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "optional": true, - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, - "node_modules/mocha": { - "version": "10.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "bin": { - "_mocha": "bin/_mocha", - "mocha": "bin/mocha.js" - }, - "engines": { - "node": ">= 14.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mochajs" - } - }, - "node_modules/mocha/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/mocha/node_modules/debug/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/mocha/node_modules/ms": { - "version": "2.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/moment": { - "version": "2.29.4", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/moment-timezone": { - "version": "0.5.45", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", - "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", - "dependencies": { - "moment": "^2.29.4" - }, - "engines": { - "node": "*" - } - }, - "node_modules/ms": { - "version": "2.0.0", - "license": "MIT" - }, - "node_modules/multiparty": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/multiparty/-/multiparty-4.2.1.tgz", - "integrity": "sha512-AvESCnNoQlZiOfP9R4mxN8M9csy2L16EIbWIkt3l4FuGti9kXBS8QVzlfyg4HEnarJhrzZilgNFlZtqmoiAIIA==", - "dependencies": { - "fd-slicer": "1.1.0", - "http-errors": "~1.7.0", - "safe-buffer": "5.1.2", - "uid-safe": "2.1.5" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/multiparty/node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/multiparty/node_modules/http-errors": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz", - "integrity": "sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/multiparty/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/multiparty/node_modules/setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "node_modules/multiparty/node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/multiparty/node_modules/toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", - "optional": true - }, - "node_modules/nanoid": { - "version": "3.3.3", - "dev": true, - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "optional": true - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "node_modules/negotiator": { - "version": "0.6.3", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/nested-error-stacks": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.1.tgz", - "integrity": "sha512-9iN1ka/9zmX1ZvLV9ewJYEk9h7RyRRtqdK0woXcqohu8EWIerfPUjYJPg0ULy0UqP7cslmdGc8xKDJcojlKiaw==" - }, - "node_modules/nise": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.9.tgz", - "integrity": "sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/text-encoding": "^0.7.2", - "just-extend": "^6.2.0", - "path-to-regexp": "^6.2.1" - } - }, - "node_modules/nise/node_modules/path-to-regexp": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", - "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==", - "dev": true - }, - "node_modules/nock": { - "version": "13.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.21", - "propagate": "^2.0.0" - }, - "engines": { - "node": ">= 10.13" - } - }, - "node_modules/nock/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/nock/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/node-abi": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", - "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==", - "optional": true, - "dependencies": { - "semver": "^5.4.1" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-preload": { - "version": "0.2.1", - "dev": true, - "license": "MIT", - "dependencies": { - "process-on-spawn": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/node-releases": { - "version": "2.0.10", - "dev": true, - "license": "MIT" - }, - "node_modules/node-sql-parser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/node-sql-parser/-/node-sql-parser-5.2.0.tgz", - "integrity": "sha512-lO/9ox0fLl47ksqlwGrorXm8BD+5UbxvUhj6T1XiCXnvoVwWWn7gC2l396p9OXNii19dc4MkyTN3XeJGNUZ8jw==", - "dependencies": { - "@types/pegjs": "^0.10.0", - "big-integer": "^1.6.48" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nodemon": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.1.tgz", - "integrity": "sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==", - "dev": true, - "dependencies": { - "chokidar": "^3.5.2", - "debug": "^3.2.7", - "ignore-by-default": "^1.0.1", - "minimatch": "^3.1.2", - "pstree.remy": "^1.1.8", - "semver": "^7.5.3", - "simple-update-notifier": "^2.0.0", - "supports-color": "^5.5.0", - "touch": "^3.1.0", - "undefsafe": "^2.0.5" - }, - "bin": { - "nodemon": "bin/nodemon.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/nodemon" - } - }, - "node_modules/nodemon/node_modules/debug": { - "version": "3.2.7", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/nodemon/node_modules/has-flag": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/nodemon/node_modules/minimatch": { - "version": "3.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/nodemon/node_modules/ms": { - "version": "2.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/nodemon/node_modules/supports-color": { - "version": "5.5.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha512-6kM8CLXvuW5crTxsAtva2YLrRrDaiTIkIePWs9moLHqbFWT94WpNFjwS/5dfLfECg5i/lkmw3aoqVidxt23TEQ==", - "optional": true - }, - "node_modules/nopt": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "abbrev": "1" - }, - "bin": { - "nopt": "bin/nopt.js" - } - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "optional": true, - "dependencies": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "node_modules/number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/nyc": { - "version": "15.1.0", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "caching-transform": "^4.0.0", - "convert-source-map": "^1.7.0", - "decamelize": "^1.2.0", - "find-cache-dir": "^3.2.0", - "find-up": "^4.1.0", - "foreground-child": "^2.0.0", - "get-package-type": "^0.1.0", - "glob": "^7.1.6", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-hook": "^3.0.0", - "istanbul-lib-instrument": "^4.0.0", - "istanbul-lib-processinfo": "^2.0.2", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.0.2", - "make-dir": "^3.0.0", - "node-preload": "^0.2.1", - "p-map": "^3.0.0", - "process-on-spawn": "^1.0.0", - "resolve-from": "^5.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "spawn-wrap": "^2.0.0", - "test-exclude": "^6.0.0", - "yargs": "^15.0.2" - }, - "bin": { - "nyc": "bin/nyc.js" - }, - "engines": { - "node": ">=8.9" - } - }, - "node_modules/nyc/node_modules/camelcase": { - "version": "5.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/nyc/node_modules/cliui": { - "version": "6.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "node_modules/nyc/node_modules/decamelize": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/nyc/node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/nyc/node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/wrap-ansi": { - "version": "6.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/y18n": { - "version": "4.0.3", - "dev": true, - "license": "ISC" - }, - "node_modules/nyc/node_modules/yargs": { - "version": "15.4.1", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/yargs-parser": { - "version": "18.1.3", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/object-hash": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", - "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==", - "engines": { - "node": ">= 6" - } - }, - "node_modules/object-inspect": { - "version": "1.12.3", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, - "node_modules/optional": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", - "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" - }, - "node_modules/optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "dependencies": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-map": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "aggregate-error": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/package-hash": { - "version": "4.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "graceful-fs": "^4.1.15", - "hasha": "^5.0.0", - "lodash.flattendeep": "^4.4.0", - "release-zalgo": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "license": "MIT" - }, - "node_modules/path-to-regexp": { - "version": "0.1.7", - "license": "MIT" - }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/pathval": { - "version": "1.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" - }, - "node_modules/pg": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", - "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", - "dependencies": { - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "engines": { - "node": ">= 8.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.1.1" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.5.0", - "license": "MIT" - }, - "node_modules/pg-hstore": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", - "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", - "dependencies": { - "underscore": "^1.13.1" - }, - "engines": { - "node": ">= 0.8.x" - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "peerDependencies": { - "pg": ">=8.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", - "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pg/node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, - "node_modules/pgpass": { - "version": "1.0.5", - "license": "MIT", - "dependencies": { - "split2": "^4.1.0" - } - }, - "node_modules/picocolors": { - "version": "1.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pkg-dir/node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/prebuild-install": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", - "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", - "optional": true, - "dependencies": { - "detect-libc": "^1.0.3", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "napi-build-utils": "^1.0.1", - "node-abi": "^2.7.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.2.7", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "license": "MIT" - }, - "node_modules/process-on-spawn": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "fromentries": "^1.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/prom-client": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.2.0.tgz", - "integrity": "sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA==", - "dependencies": { - "tdigest": "^0.1.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/propagate": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "license": "MIT" - }, - "node_modules/pstree.remy": { - "version": "1.1.8", - "dev": true, - "license": "MIT" - }, - "node_modules/pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/punycode": { - "version": "2.3.0", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/qs": { - "version": "6.11.0", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.0.4" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/random-bytes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/randombytes": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "optional": true, - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc/node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/readable-stream": { - "version": "2.3.8", - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/readable-stream/node_modules/safe-buffer": { - "version": "5.1.2", - "license": "MIT" - }, - "node_modules/readdirp": { - "version": "3.6.0", - "dev": true, - "license": "MIT", - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/rechoir": { - "version": "0.8.0", - "license": "MIT", - "dependencies": { - "resolve": "^1.20.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/release-zalgo": { - "version": "1.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "es6-error": "^4.0.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/resolve": { - "version": "1.22.1", - "license": "MIT", - "dependencies": { - "is-core-module": "^2.9.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "engines": { - "node": ">= 4" - } - }, - "node_modules/retry-as-promised": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-7.0.4.tgz", - "integrity": "sha512-XgmCoxKWkDofwH8WddD0w85ZfqYz+ZHlr5yo+3YUCfycWawU56T5ckWXsScsj5B8tqUcIG67DxXByo3VUgiAdA==" - }, - "node_modules/retry-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", - "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", - "dependencies": { - "@types/request": "^2.48.8", - "extend": "^3.0.2", - "teeny-request": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/rimraf": { - "version": "3.0.2", - "dev": true, - "license": "ISC", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safe-stable-stringify": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", - "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", - "engines": { - "node": ">=10" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" - }, - "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/send": { - "version": "0.18.0", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "license": "MIT" - }, - "node_modules/sequelize": { - "version": "6.37.3", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.3.tgz", - "integrity": "sha512-V2FTqYpdZjPy3VQrZvjTPnOoLm0KudCRXfGWp48QwhyPPp2yW8z0p0sCYZd/em847Tl2dVxJJ1DR+hF+O77T7A==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/sequelize" - } - ], - "dependencies": { - "@types/debug": "^4.1.8", - "@types/validator": "^13.7.17", - "debug": "^4.3.4", - "dottie": "^2.0.6", - "inflection": "^1.13.4", - "lodash": "^4.17.21", - "moment": "^2.29.4", - "moment-timezone": "^0.5.43", - "pg-connection-string": "^2.6.1", - "retry-as-promised": "^7.0.4", - "semver": "^7.5.4", - "sequelize-pool": "^7.1.0", - "toposort-class": "^1.0.1", - "uuid": "^8.3.2", - "validator": "^13.9.0", - "wkx": "^0.5.0" - }, - "engines": { - "node": ">=10.0.0" - }, - "peerDependenciesMeta": { - "ibm_db": { - "optional": true - }, - "mariadb": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "oracledb": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-hstore": { - "optional": true - }, - "snowflake-sdk": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, - "node_modules/sequelize-pool": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz", - "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg==", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/sequelize/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/sequelize/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/sequelize/node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, - "node_modules/sequelize/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/serialize-javascript": { - "version": "6.0.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" - } - }, - "node_modules/serve-static": { - "version": "1.15.0", - "license": "MIT", - "dependencies": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.18.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/set-blocking": { - "version": "2.0.0", - "devOptional": true, - "license": "ISC" - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "license": "ISC" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/side-channel": { - "version": "1.0.4", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "devOptional": true, - "license": "ISC" - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "optional": true - }, - "node_modules/simple-get": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", - "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", - "optional": true, - "dependencies": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/simple-update-notifier": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", - "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", - "dev": true, - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sinon": { - "version": "17.0.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", - "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.5", - "supports-color": "^7.2.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/sinon" - } - }, - "node_modules/sinon/node_modules/diff": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", - "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/sinon/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/slug": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/slug/-/slug-9.1.0.tgz", - "integrity": "sha512-ioOsCfzQSu+D6NZ8XMCR8IW9FgvF8W7Xzz56hBkB/ALvNaWeBs2MUvvPugq3GCrxfHPFeK6hAxGkY/WLnfX2Lg==", - "bin": { - "slug": "cli.js" - } - }, - "node_modules/snappy": { - "version": "6.3.5", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", - "integrity": "sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA==", - "hasInstallScript": true, - "optional": true, - "dependencies": { - "bindings": "^1.3.1", - "nan": "^2.14.1", - "prebuild-install": "5.3.0" - } - }, - "node_modules/snappyjs": { - "version": "0.6.1", - "license": "MIT" - }, - "node_modules/source-map": { - "version": "0.6.1", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/spawn-wrap": { - "version": "2.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^2.0.0", - "is-windows": "^1.0.2", - "make-dir": "^3.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "which": "^2.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/split2": { - "version": "4.1.0", - "license": "ISC", - "engines": { - "node": ">= 10.x" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", - "engines": { - "node": "*" - } - }, - "node_modules/statuses": { - "version": "2.0.1", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/stream-browserify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", - "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", - "dependencies": { - "inherits": "~2.0.4", - "readable-stream": "^3.5.0" - } - }, - "node_modules/stream-browserify/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "dependencies": { - "stubs": "^3.0.0" - } - }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/string_decoder/node_modules/safe-buffer": { - "version": "5.1.2", - "license": "MIT" - }, - "node_modules/string-width": { - "version": "4.2.3", - "devOptional": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "devOptional": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, - "node_modules/stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" - }, - "node_modules/superagent": { - "version": "3.8.3", - "dev": true, - "license": "MIT", - "dependencies": { - "component-emitter": "^1.2.0", - "cookiejar": "^2.1.0", - "debug": "^3.1.0", - "extend": "^3.0.0", - "form-data": "^2.3.1", - "formidable": "^1.2.0", - "methods": "^1.1.1", - "mime": "^1.4.1", - "qs": "^6.5.1", - "readable-stream": "^2.3.5" - }, - "engines": { - "node": ">= 4.0" - } - }, - "node_modules/superagent/node_modules/debug": { - "version": "3.2.7", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/superagent/node_modules/form-data": { - "version": "2.5.1", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/superagent/node_modules/ms": { - "version": "2.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "8.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "optional": true, - "dependencies": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - } - }, - "node_modules/tar-fs/node_modules/pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "optional": true, - "dependencies": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/tar-stream/node_modules/bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", - "optional": true, - "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "node_modules/tarn": { - "version": "3.0.2", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", - "dependencies": { - "bintrees": "1.0.2" - } - }, - "node_modules/teeny-request": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", - "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.9", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/teeny-request/node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/teeny-request/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, - "node_modules/text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true - }, - "node_modules/tildify": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", - "optional": true - }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/toposort-class": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", - "integrity": "sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg==" - }, - "node_modules/touch": { - "version": "3.1.0", - "dev": true, - "license": "ISC", - "dependencies": { - "nopt": "~1.0.10" - }, - "bin": { - "nodetouch": "bin/nodetouch.js" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", - "engines": { - "node": "*" - } - }, - "node_modules/trino-client": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/trino-client/-/trino-client-0.2.2.tgz", - "integrity": "sha512-TMndAbFiiGlAbJotsqsbMSqFZ8sPtE+KBIl/qYEAI9+eAXruNvz0BhxFC67PE39eouVn+JtF6L8Yr2q6/sAlAA==", - "dependencies": { - "axios": "1.6.2" - } - }, - "node_modules/triple-beam": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", - "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", - "dev": true, - "engines": { - "node": ">=16" - }, - "peerDependencies": { - "typescript": ">=4.2.0" - } - }, - "node_modules/ts-node": { - "version": "10.9.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/ts-node/node_modules/diff": { - "version": "4.0.2", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/tsconfig-paths": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "json5": "^2.2.2", - "minimist": "^1.2.6", - "strip-bom": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/tsconfig-paths/node_modules/strip-bom": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "optional": true, - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/type-detect": { - "version": "4.0.8", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.8.1", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=8" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "license": "MIT", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/typedarray-to-buffer": { - "version": "3.1.5", - "dev": true, - "license": "MIT", - "dependencies": { - "is-typedarray": "^1.0.0" - } - }, - "node_modules/typescript": { - "version": "4.9.5", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, - "node_modules/uid-safe": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", - "dependencies": { - "random-bytes": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/undefsafe": { - "version": "2.0.5", - "dev": true, - "license": "MIT" - }, - "node_modules/underscore": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", - "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" - }, - "node_modules/unpipe": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.0.10", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" - }, - "bin": { - "browserslist-lint": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, - "node_modules/url/node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" - }, - "node_modules/util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "license": "MIT" - }, - "node_modules/utils-merge": { - "version": "1.0.1", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/validator": { - "version": "13.12.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.12.0.tgz", - "integrity": "sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/which-module": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/which-pm-runs": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.1.0.tgz", - "integrity": "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==", - "optional": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/wide-align": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", - "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "optional": true, - "dependencies": { - "string-width": "^1.0.2 || 2 || 3 || 4" - } - }, - "node_modules/winston": { - "version": "2.4.7", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz", - "integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==", - "dependencies": { - "async": "^2.6.4", - "colors": "1.0.x", - "cycle": "1.0.x", - "eyes": "0.1.x", - "isstream": "0.1.x", - "stack-trace": "0.0.x" - }, - "engines": { - "node": ">= 0.10.0" - } - }, - "node_modules/winston-compat": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/winston-compat/-/winston-compat-0.1.5.tgz", - "integrity": "sha512-EPvPcHT604AV3Ji6d3+vX8ENKIml9VSxMRnPQ+cuK/FX6f3hvPP2hxyoeeCOCFvDrJEujalfcKWlWPvAnFyS9g==", - "dependencies": { - "cycle": "~1.0.3", - "logform": "^1.6.0", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-daily-rotate-file": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-3.2.3.tgz", - "integrity": "sha512-BOvmvQH2WaiexOjzj14YNHSc18IDyZJ9t4pMsbTERjpjMltoBVijM8DDJJPr2jSqELSNnbgGPBk3kDQSRgOAtQ==", - "dependencies": { - "file-stream-rotator": "^0.2.1", - "semver": "^5.5.0", - "triple-beam": "^1.3.0", - "winston-compat": "^0.1.4", - "winston-transport": "^4.2.0" - }, - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "winston": "^2 || ^3" - } - }, - "node_modules/winston-transport": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.0.tgz", - "integrity": "sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==", - "dependencies": { - "logform": "^2.3.2", - "readable-stream": "^3.6.0", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/winston-transport/node_modules/@colors/colors": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", - "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/winston-transport/node_modules/fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "node_modules/winston-transport/node_modules/logform": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.5.1.tgz", - "integrity": "sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==", - "dependencies": { - "@colors/colors": "1.5.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - } - }, - "node_modules/winston-transport/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/wkx": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.5.0.tgz", - "integrity": "sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg==", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/workerpool": { - "version": "6.2.1", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "license": "ISC" - }, - "node_modules/write-file-atomic": { - "version": "3.0.3", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "is-typedarray": "^1.0.0", - "signal-exit": "^3.0.2", - "typedarray-to-buffer": "^3.1.5" - } - }, - "node_modules/xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "16.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.4", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-unparser": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "camelcase": "^6.0.0", - "decamelize": "^4.0.0", - "flat": "^5.0.2", - "is-plain-obj": "^2.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yn": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - }, - "dependencies": { - "@ampproject/remapping": { - "version": "2.2.0", - "dev": true, - "requires": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - } - }, - "@aws-crypto/crc32": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", - "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", - "requires": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/crc32c": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz", - "integrity": "sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==", - "requires": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/ie11-detection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", - "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", - "requires": { - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/sha1-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz", - "integrity": "sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==", - "requires": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/sha256-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", - "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", - "requires": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/sha256-js": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/sha256-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", - "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", - "requires": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/supports-web-crypto": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", - "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", - "requires": { - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/util": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", - "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", - "requires": { - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-sdk/client-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.427.0.tgz", - "integrity": "sha512-9brRaNnl6haE7R3R43A5CSNw0k1YtB3xjuArbMg/p6NDUpvRSRgOVNWu2R02Yjh/j2ZuaLOCPLuCipb+PHQPKQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/client-s3": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.590.0.tgz", - "integrity": "sha512-so+pNua0ihsHaSdskw8HCwruoYTAfYSEs3ix4GD1++83C96KaJp3udAutYiCA+84JXg9zitFa7eK7ORJAVZmTw==", - "requires": { - "@aws-crypto/sha1-browser": "3.0.0", - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-bucket-endpoint": "3.587.0", - "@aws-sdk/middleware-expect-continue": "3.577.0", - "@aws-sdk/middleware-flexible-checksums": "3.587.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-location-constraint": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/middleware-signing": "3.587.0", - "@aws-sdk/middleware-ssec": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@aws-sdk/xml-builder": "3.575.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/eventstream-serde-browser": "^3.0.0", - "@smithy/eventstream-serde-config-resolver": "^3.0.0", - "@smithy/eventstream-serde-node": "^3.0.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-blob-browser": "^3.0.0", - "@smithy/hash-node": "^3.0.0", - "@smithy/hash-stream-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/md5-js": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "@smithy/util-utf8": "^3.0.0", - "@smithy/util-waiter": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "requires": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-signing": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.587.0.tgz", - "integrity": "sha512-tiZaTDj4RvhXGRAlncFn7CSEfL3iNPO67WSaxAq+Ls5j1VgczPhu5262cWONNoMgth3nXR1hhLC4ITSl/a6AzA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "requires": { - "@smithy/types": "^3.0.0" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "requires": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "requires": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "@aws-sdk/client-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.427.0.tgz", - "integrity": "sha512-sFVFEmsQ1rmgYO1SgrOTxE/MTKpeE4hpOkm1WqhLQK7Ij136vXpjCxjH1JYZiHiUzO1wr9t4ex4dlB5J3VS/Xg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/client-sso-oidc": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.590.0.tgz", - "integrity": "sha512-3yCLPjq6WFfDpdUJKk/gSz4eAPDTjVknXaveMPi2QoVBCshneOnJsV16uNKlpVF1frTHrrDRfKYmbaVh6nFBvQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "requires": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "requires": { - "@smithy/types": "^3.0.0" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "requires": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "requires": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "@aws-sdk/client-sts": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.427.0.tgz", - "integrity": "sha512-le2wLJKILyWuRfPz2HbyaNtu5kEki+ojUkTqCU6FPDRrqUvEkaaCBH9Awo/2AtrCfRkiobop8RuTTj6cAnpiJg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-sdk-sts": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/core": { - "version": "3.588.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.588.0.tgz", - "integrity": "sha512-O1c2+9ce46Z+iiid+W3iC1IvPbfIo5ev9CBi54GdNB9SaI8/3+f8MJcux0D6c9toCF0ArMersN/gp8ek57e9uQ==", - "requires": { - "@smithy/core": "^2.1.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/credential-provider-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.427.0.tgz", - "integrity": "sha512-BQNzNrMJlBAfXhYNdAUqaVASpT9Aho5swj7glZKxx4Uds1w5Pih2e14JWgnl8XgUWAZ36pchTrV1aA4JT7N8vw==", - "requires": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-env": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.425.0.tgz", - "integrity": "sha512-J20etnLvMKXRVi5FK4F8yOCNm2RTaQn5psQTGdDEPWJNGxohcSpzzls8U2KcMyUJ+vItlrThr4qwgpHG3i/N0w==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-http": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.425.0.tgz", - "integrity": "sha512-aP9nkoVWf+OlNMecrUqe4+RuQrX13nucVbty0HTvuwfwJJj0T6ByWZzle+fo1D+5OxvJmtzTflBWt6jUERdHWA==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-ini": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.427.0.tgz", - "integrity": "sha512-NmH1cO/w98CKMltYec3IrJIIco19wRjATFNiw83c+FGXZ+InJwReqBnruxIOmKTx2KDzd6fwU1HOewS7UjaaaQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-node": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.427.0.tgz", - "integrity": "sha512-wYYbQ57nKL8OfgRbl8k6uXcdnYml+p3LSSfDUAuUEp1HKlQ8lOXFJ3BdLr5qrk7LhpyppSRnWBmh2c3kWa7ANQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-process": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.425.0.tgz", - "integrity": "sha512-YY6tkLdvtb1Fgofp3b1UWO+5vwS14LJ/smGmuGpSba0V7gFJRdcrJ9bcb9vVgAGuMdjzRJ+bUKlLLtqXkaykEw==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.427.0.tgz", - "integrity": "sha512-c+tXyS/i49erHs4bAp6vKNYeYlyQ0VNMBgoco0LCn1rL0REtHbfhWMnqDLF6c2n3yIWDOTrQu0D73Idnpy16eA==", - "requires": { - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/token-providers": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-web-identity": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.425.0.tgz", - "integrity": "sha512-/0R65TgRzL01JU3SzloivWNwdkbIhr06uY/F5pBHf/DynQqaspKNfdHn6AiozgSVDfwRHFjKBTUy6wvf3QFkuA==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.427.0.tgz", - "integrity": "sha512-rKKohSHju462vo+uQnPjcEZPBAfAMgGH6K1XyyCNpuOC0yYLkG87PYpvAQeb8riTrkHPX0dYUHuTHZ6zQgMGjA==", - "requires": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-cognito-identity": "3.427.0", - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-http": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/lib-storage": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.427.0.tgz", - "integrity": "sha512-JE26Zo4SMMY2SGlD/FilF5MpLuP8D2IrW+ye/J77dwh91gyGnNa/lubJ7WbVjIAxgeaDHsAkcpNO4VR5k6JCKg==", - "requires": { - "@smithy/abort-controller": "^2.0.1", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/smithy-client": "^2.1.9", - "buffer": "5.6.0", - "events": "3.3.0", - "stream-browserify": "3.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-bucket-endpoint": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.587.0.tgz", - "integrity": "sha512-HkFXLPl8pr6BH/Q0JpOESqEKL0ZK3sk7aSZ1S6GE4RXET7H5R94THULXqQFZzD48gZcyFooO/yNKZTqrZFaWKg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-expect-continue": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.577.0.tgz", - "integrity": "sha512-6dPp8Tv4F0of4un5IAyG6q++GrRrNQQ4P2NAMB1W0VO4JoEu1C8GievbbDLi88TFIFmtKpnHB0ODCzwnoe8JsA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-flexible-checksums": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.587.0.tgz", - "integrity": "sha512-URMwp/budDvKhIvZ4a6zIBfFTun/iDlPWXqsGKYjEtHt8jz27OSjCZtDtIeqW4WTBdKL8KZgQcl+DdaE5M1qiQ==", - "requires": { - "@aws-crypto/crc32": "3.0.0", - "@aws-crypto/crc32c": "3.0.0", - "@aws-sdk/types": "3.577.0", - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-host-header": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.425.0.tgz", - "integrity": "sha512-E5Gt41LObQ+cr8QnLthwsH3MtVSNXy1AKJMowDr85h0vzqA/FHUkgHyOGntgozzjXT5M0MaSRYxS0xwTR5D4Ew==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-location-constraint": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.577.0.tgz", - "integrity": "sha512-DKPTD2D2s+t2QUo/IXYtVa/6Un8GZ+phSTBkyBNx2kfZz4Kwavhl/JJzSqTV3GfCXkVdFu7CrjoX7BZ6qWeTUA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-logger": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.425.0.tgz", - "integrity": "sha512-INE9XWRXx2f4a/r2vOU0tAmgctVp7nEaEasemNtVBYhqbKLZvr9ndLBSgKGgJ8LIcXAoISipaMuFiqIGkFsm7A==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-recursion-detection": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.425.0.tgz", - "integrity": "sha512-77gnzJ5b91bgD75L/ugpOyerx6lR3oyS4080X1YI58EzdyBMkDrHM4FbMcY2RynETi3lwXCFzLRyZjWXY1mRlw==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-sdk-s3": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.587.0.tgz", - "integrity": "sha512-vtXTGEiw1E9Fax4LmcU2Z208gbrC8ShrdsSLmGcRPpu5NPOGBFBSDG5sy5EDNClrFxIl/Le8coQnD0EDBtx+uQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-sdk-sts": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.425.0.tgz", - "integrity": "sha512-JFojrg76oKAoBknnr9EL5N2aJ1mRCtBqXoZYST58GSx8uYdFQ89qS65VNQ8JviBXzsrCNAn4vDhZ5Ch5E6TxGQ==", - "requires": { - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-signing": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.425.0.tgz", - "integrity": "sha512-ZpOfgJHk7ovQ0sSwg3tU4NxFOnz53lJlkJRf7S+wxQALHM0P2MJ6LYBrZaFMVsKiJxNIdZBXD6jclgHg72ZW6Q==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/signature-v4": "^2.0.0", - "@smithy/types": "^2.3.4", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-ssec": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.577.0.tgz", - "integrity": "sha512-i2BPJR+rp8xmRVIGc0h1kDRFcM2J9GnClqqpc+NLSjmYadlcg4mPklisz9HzwFVcRPJ5XcGf3U4BYs5G8+iTyg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-user-agent": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.427.0.tgz", - "integrity": "sha512-y9HxYsNvnA3KqDl8w1jHeCwz4P9CuBEtu/G+KYffLeAMBsMZmh4SIkFFCO9wE/dyYg6+yo07rYcnnIfy7WA0bw==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/region-config-resolver": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.425.0.tgz", - "integrity": "sha512-u7uv/iUOapIJdRgRkO3wnpYsUgV6ponsZJQgVg/8L+n+Vo5PQL5gAcIuAOwcYSKQPFaeK+KbmByI4SyOK203Vw==", - "requires": { - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/s3-request-presigner": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.590.0.tgz", - "integrity": "sha512-bb8NEG2IUHqFQJsLzr1nlkTZYyokeo3bGbHwMBKZHbdF+OXrQx0kQUcaDCXYWmeydSfHXxweQEJ2U5i1YEvT/A==", - "requires": { - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-format-url": "3.577.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/signature-v4-multi-region": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.587.0.tgz", - "integrity": "sha512-TR9+ZSjdXvXUz54ayHcCihhcvxI9W7102J1OK6MrLgBlPE7uRhAx42BR9L5lLJ86Xj3LuqPWf//o9d/zR9WVIg==", - "requires": { - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/token-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.427.0.tgz", - "integrity": "sha512-4E5E+4p8lJ69PBY400dJXF06LUHYx5lkKzBEsYqWWhoZcoftrvi24ltIhUDoGVLkrLcTHZIWSdFAWSos4hXqeg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/types": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.425.0.tgz", - "integrity": "sha512-6lqbmorwerN4v+J5dqbHPAsjynI0mkEF+blf+69QTaKKGaxBBVaXgqoqul9RXYcK5MMrrYRbQIMd0zYOoy90kA==", - "requires": { - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-arn-parser": { - "version": "3.568.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", - "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-endpoints": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.427.0.tgz", - "integrity": "sha512-rSyiAIFF/EVvity/+LWUqoTMJ0a25RAc9iqx0WZ4tf1UjuEXRRXxZEb+jEZg1bk+pY84gdLdx9z5E+MSJCZxNQ==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-format-url": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.577.0.tgz", - "integrity": "sha512-SyEGC2J+y/krFRuPgiF02FmMYhqbiIkOjDE6k4nYLJQRyS6XEAGxZoG+OHeOVEM+bsDgbxokXZiM3XKGu6qFIg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/util-locate-window": { - "version": "3.310.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.310.0.tgz", - "integrity": "sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-user-agent-browser": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.425.0.tgz", - "integrity": "sha512-22Y9iMtjGcFjGILR6/xdp1qRezlHVLyXtnpEsbuPTiernRCPk6zfAnK/ATH77r02MUjU057tdxVkd5umUBTn9Q==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-user-agent-node": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.425.0.tgz", - "integrity": "sha512-SIR4F5uQeeVAi8lv4OgRirtdtNi5zeyogTuQgGi9su8F/WP1N6JqxofcwpUY5f8/oJ2UlXr/tx1f09UHfJJzvA==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-utf8-browser": { - "version": "3.259.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", - "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", - "requires": { - "tslib": "^2.3.1" - } - }, - "@aws-sdk/xml-builder": { - "version": "3.575.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.575.0.tgz", - "integrity": "sha512-cWgAwmbFYNCFzPwxL705+lWps0F3ZvOckufd2KKoEZUmtpVw9/txUXNrPySUXSmRTSRhoatIMABNfStWR043bQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/abort-controller": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", - "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", - "requires": { - "tslib": "^2.2.0" - } - }, - "@azure/core-auth": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.7.2.tgz", - "integrity": "sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.1.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-client": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", - "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-rest-pipeline": "^1.9.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/logger": "^1.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-http-compat": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz", - "integrity": "sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-client": "^1.3.0", - "@azure/core-rest-pipeline": "^1.3.0" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-lro": { - "version": "2.5.4", - "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.4.tgz", - "integrity": "sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q==", - "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-util": "^1.2.0", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" - } - }, - "@azure/core-paging": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", - "integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==", - "requires": { - "tslib": "^2.2.0" - } - }, - "@azure/core-rest-pipeline": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.0.tgz", - "integrity": "sha512-CeuTvsXxCUmEuxH5g/aceuSl6w2EugvNHKAtKKVdiX915EjJJxAwfzNNWZreNnbxHZ2fi0zaM6wwS23x2JVqSQ==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.9.0", - "@azure/logger": "^1.0.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-tracing": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.2.tgz", - "integrity": "sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@azure/core-util": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.9.0.tgz", - "integrity": "sha512-AfalUQ1ZppaKuxPPMsFEUdX6GZPB3d9paR9d/TTL7Ow2De8cJaC7ibi7kWVlFAVPCYo31OcnGymc0R89DX8Oaw==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-xml": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.2.tgz", - "integrity": "sha512-CW3MZhApe/S4iikbYKE7s83fjDBPIr2kpidX+hlGRwh7N4o1nIpQ/PfJTeioqhfqdMvRtheEl+ft64fyTaLNaA==", - "requires": { - "fast-xml-parser": "^4.3.2", - "tslib": "^2.6.2" - }, - "dependencies": { - "fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "requires": { - "strnum": "^1.0.5" - } - } - } - }, - "@azure/logger": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz", - "integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==", - "requires": { - "tslib": "^2.2.0" - } - }, - "@azure/storage-blob": { - "version": "12.23.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.23.0.tgz", - "integrity": "sha512-c1KJ5R5hqR/HtvmFtTn/Y1BNMq45NUBp0LZH7yF8WFMET+wmESgEr0FVTu/Z5NonmfUjbgJZG5Nh8xHc5RdWGQ==", - "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-client": "^1.6.2", - "@azure/core-http-compat": "^2.0.0", - "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-rest-pipeline": "^1.10.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/core-xml": "^1.3.2", - "@azure/logger": "^1.0.0", - "events": "^3.0.0", - "tslib": "^2.2.0" - } - }, - "@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", - "dev": true, - "requires": { - "@babel/highlight": "^7.22.13", - "chalk": "^2.4.2" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "@babel/compat-data": { - "version": "7.21.0", - "dev": true - }, - "@babel/core": { - "version": "7.21.3", - "dev": true, - "requires": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.21.3", - "@babel/helper-compilation-targets": "^7.20.7", - "@babel/helper-module-transforms": "^7.21.2", - "@babel/helpers": "^7.21.0", - "@babel/parser": "^7.21.3", - "@babel/template": "^7.20.7", - "@babel/traverse": "7.23.2", - "@babel/types": "^7.21.3", - "convert-source-map": "^1.7.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.2", - "semver": "^7.5.3" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "@babel/generator": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.3.tgz", - "integrity": "sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg==", - "dev": true, - "requires": { - "@babel/types": "^7.23.3", - "@jridgewell/gen-mapping": "^0.3.2", - "@jridgewell/trace-mapping": "^0.3.17", - "jsesc": "^2.5.1" - }, - "dependencies": { - "@jridgewell/gen-mapping": { - "version": "0.3.2", - "dev": true, - "requires": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - } - } - } - }, - "@babel/helper-compilation-targets": { - "version": "7.20.7", - "dev": true, - "requires": { - "@babel/compat-data": "^7.20.5", - "@babel/helper-validator-option": "^7.18.6", - "browserslist": "^4.21.3", - "lru-cache": "^5.1.1", - "semver": "^7.5.3" - } - }, - "@babel/helper-environment-visitor": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", - "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", - "dev": true - }, - "@babel/helper-function-name": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", - "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", - "dev": true, - "requires": { - "@babel/template": "^7.22.15", - "@babel/types": "^7.23.0" - } - }, - "@babel/helper-hoist-variables": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", - "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", - "dev": true, - "requires": { - "@babel/types": "^7.22.5" - } - }, - "@babel/helper-module-imports": { - "version": "7.18.6", - "dev": true, - "requires": { - "@babel/types": "^7.18.6" - } - }, - "@babel/helper-module-transforms": { - "version": "7.21.2", - "dev": true, - "requires": { - "@babel/helper-environment-visitor": "^7.18.9", - "@babel/helper-module-imports": "^7.18.6", - "@babel/helper-simple-access": "^7.20.2", - "@babel/helper-split-export-declaration": "^7.18.6", - "@babel/helper-validator-identifier": "^7.19.1", - "@babel/template": "^7.20.7", - "@babel/traverse": "7.23.2", - "@babel/types": "^7.21.2" - } - }, - "@babel/helper-simple-access": { - "version": "7.20.2", - "dev": true, - "requires": { - "@babel/types": "^7.20.2" - } - }, - "@babel/helper-split-export-declaration": { - "version": "7.22.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", - "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", - "dev": true, - "requires": { - "@babel/types": "^7.22.5" - } - }, - "@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", - "dev": true - }, - "@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", - "dev": true - }, - "@babel/helper-validator-option": { - "version": "7.21.0", - "dev": true - }, - "@babel/helpers": { - "version": "7.21.0", - "dev": true, - "requires": { - "@babel/template": "^7.20.7", - "@babel/traverse": "7.23.2", - "@babel/types": "^7.21.0" - } - }, - "@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "@babel/parser": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.3.tgz", - "integrity": "sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw==", - "dev": true - }, - "@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" - } - }, - "@babel/traverse": { - "version": "7.23.2", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", - "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.0", - "@babel/helper-environment-visitor": "^7.22.20", - "@babel/helper-function-name": "^7.23.0", - "@babel/helper-hoist-variables": "^7.22.5", - "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.0", - "@babel/types": "^7.23.0", - "debug": "^4.1.0", - "globals": "^11.1.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "@babel/types": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.3.tgz", - "integrity": "sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw==", - "dev": true, - "requires": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" - } - }, - "@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==" - }, - "@cspotcode/source-map-support": { - "version": "0.8.1", - "dev": true, - "requires": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "dependencies": { - "@jridgewell/trace-mapping": { - "version": "0.3.9", - "dev": true, - "requires": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - } - } - }, - "@dabh/diagnostics": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", - "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", - "requires": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, - "@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^3.3.0" - } - }, - "@eslint-community/regexpp": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.1.tgz", - "integrity": "sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA==", - "dev": true - }, - "@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", - "dev": true, - "requires": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "dependencies": { - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "requires": { - "type-fest": "^0.20.2" - } - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true - } - } - }, - "@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", - "dev": true - }, - "@google-cloud/paginator": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", - "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", - "requires": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - } - }, - "@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==" - }, - "@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==" - }, - "@google-cloud/storage": { - "version": "7.11.1", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.11.1.tgz", - "integrity": "sha512-tibLSvgw7nDohMyIelt26kBpJ59YGWA2Rzep++DFNzEzKaSuCSp56Se9iM13ZlM3j5nLzR21IBkpRN58CmvCIw==", - "requires": { - "@google-cloud/paginator": "^5.0.0", - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "^4.0.0", - "abort-controller": "^3.0.0", - "async-retry": "^1.3.3", - "duplexify": "^4.1.3", - "fast-xml-parser": "^4.3.0", - "gaxios": "^6.0.2", - "google-auth-library": "^9.6.3", - "html-entities": "^2.5.2", - "mime": "^3.0.0", - "p-limit": "^3.0.1", - "retry-request": "^7.0.0", - "teeny-request": "^9.0.0", - "uuid": "^8.0.0" - }, - "dependencies": { - "fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "requires": { - "strnum": "^1.0.5" - } - }, - "mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==" - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - } - } - }, - "@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", - "dev": true, - "requires": { - "@humanwhocodes/object-schema": "^2.0.2", - "debug": "^4.3.1", - "minimatch": "^3.0.5" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true - }, - "@humanwhocodes/object-schema": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", - "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", - "dev": true - }, - "@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "dev": true, - "requires": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "dependencies": { - "argparse": { - "version": "1.0.10", - "dev": true, - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "camelcase": { - "version": "5.3.1", - "dev": true - }, - "esprima": { - "version": "4.0.1", - "dev": true - }, - "find-up": { - "version": "4.1.0", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "js-yaml": { - "version": "3.14.1", - "dev": true, - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-limit": { - "version": "2.3.0", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "sprintf-js": { - "version": "1.0.3", - "dev": true - } - } - }, - "@istanbuljs/schema": { - "version": "0.1.3", - "dev": true - }, - "@jridgewell/gen-mapping": { - "version": "0.1.1", - "dev": true, - "requires": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "@jridgewell/resolve-uri": { - "version": "3.1.0", - "dev": true - }, - "@jridgewell/set-array": { - "version": "1.1.2", - "dev": true - }, - "@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "dev": true - }, - "@jridgewell/trace-mapping": { - "version": "0.3.17", - "dev": true, - "requires": { - "@jridgewell/resolve-uri": "3.1.0", - "@jridgewell/sourcemap-codec": "1.4.14" - } - }, - "@napi-rs/snappy-android-arm-eabi": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz", - "integrity": "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==", - "optional": true - }, - "@napi-rs/snappy-android-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz", - "integrity": "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==", - "optional": true - }, - "@napi-rs/snappy-darwin-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz", - "integrity": "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==", - "optional": true - }, - "@napi-rs/snappy-darwin-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz", - "integrity": "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==", - "optional": true - }, - "@napi-rs/snappy-freebsd-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz", - "integrity": "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==", - "optional": true - }, - "@napi-rs/snappy-linux-arm-gnueabihf": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz", - "integrity": "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==", - "optional": true - }, - "@napi-rs/snappy-linux-arm64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz", - "integrity": "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==", - "optional": true - }, - "@napi-rs/snappy-linux-arm64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz", - "integrity": "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==", - "optional": true - }, - "@napi-rs/snappy-linux-x64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz", - "integrity": "sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==", - "optional": true - }, - "@napi-rs/snappy-linux-x64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.2.2.tgz", - "integrity": "sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==", - "optional": true - }, - "@napi-rs/snappy-win32-arm64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz", - "integrity": "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==", - "optional": true - }, - "@napi-rs/snappy-win32-ia32-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz", - "integrity": "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==", - "optional": true - }, - "@napi-rs/snappy-win32-x64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz", - "integrity": "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==", - "optional": true - }, - "@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - } - }, - "@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true - }, - "@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "requires": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - } - }, - "@project-sunbird/logger": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/@project-sunbird/logger/-/logger-0.0.9.tgz", - "integrity": "sha512-5DLFvgPBa5ffeavwY7fsXQsmZ+CM9rNC3Qmbh9pa4Ygfm10fhucijG0Aqku6WN9i5EeZXLoIOhDGr2fYjKsUqg==", - "requires": { - "@types/lodash": "^4.14.149", - "@types/node": "^13.9.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.15", - "winston": "^3.2.1", - "winston-daily-rotate-file": "^4.4.2" - }, - "dependencies": { - "@types/node": { - "version": "13.13.52", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.52.tgz", - "integrity": "sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ==" - }, - "async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" - }, - "fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "file-stream-rotator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", - "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", - "requires": { - "moment": "^2.29.1" - } - }, - "logform": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", - "integrity": "sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==", - "requires": { - "@colors/colors": "1.6.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, - "winston": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.0.tgz", - "integrity": "sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==", - "requires": { - "@colors/colors": "^1.6.0", - "@dabh/diagnostics": "^2.0.2", - "async": "^3.2.3", - "is-stream": "^2.0.0", - "logform": "^2.4.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "safe-stable-stringify": "^2.3.1", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.7.0" - } - }, - "winston-daily-rotate-file": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz", - "integrity": "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA==", - "requires": { - "file-stream-rotator": "^0.6.1", - "object-hash": "^2.0.1", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - } - } - } - }, - "@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "requires": { - "type-detect": "4.0.8" - } - }, - "@sinonjs/fake-timers": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", - "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0" - } - }, - "@sinonjs/samsam": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz", - "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==", - "dev": true, - "requires": { - "@sinonjs/commons": "^2.0.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - }, - "dependencies": { - "@sinonjs/commons": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", - "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", - "dev": true, - "requires": { - "type-detect": "4.0.8" - } - } - } - }, - "@sinonjs/text-encoding": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", - "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", - "dev": true - }, - "@smithy/abort-controller": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.11.tgz", - "integrity": "sha512-MSzE1qR2JNyb7ot3blIOT3O3H0Jn06iNDEgHRaqZUwBgx5EG+VIx24Y21tlKofzYryIOcWpIohLrIIyocD6LMA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/chunked-blob-reader": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", - "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/chunked-blob-reader-native": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", - "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", - "requires": { - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/config-resolver": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-2.0.14.tgz", - "integrity": "sha512-K1K+FuWQoy8j/G7lAmK85o03O89s2Vvh6kMFmzEmiHUoQCRH1rzbDtMnGNiaMHeSeYJ6y79IyTusdRG+LuWwtg==", - "requires": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/types": "^2.3.5", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - } - }, - "@smithy/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.0.tgz", - "integrity": "sha512-ygLZSSKgt9bR8HAxR9mK+U5obvAJBr6zlQuhN5soYWx/amjDoQN4dTkydTypgKe6rIbUjTILyLU+W5XFwXr4kg==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "requires": { - "@smithy/types": "^3.0.0" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "requires": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "@smithy/credential-provider-imds": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-2.0.16.tgz", - "integrity": "sha512-tKa2xF+69TvGxJT+lnJpGrKxUuAZDLYXFhqnPEgnHz+psTpkpcB4QRjHj63+uj83KaeFJdTfW201eLZeRn6FfA==", - "requires": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "tslib": "^2.5.0" - } - }, - "@smithy/eventstream-codec": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-2.0.11.tgz", - "integrity": "sha512-BQCTjxhCYRZIfXapa2LmZSaH8QUBGwMZw7XRN83hrdixbLjIcj+o549zjkedFS07Ve2TlvWUI6BTzP+nv7snBA==", - "requires": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/eventstream-serde-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.0.tgz", - "integrity": "sha512-NB7AFiPN4NxP/YCAnrvYR18z2/ZsiHiF7VtG30gshO9GbFrIb1rC8ep4NGpJSWrz6P64uhPXeo4M0UsCLnZKqw==", - "requires": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/eventstream-serde-config-resolver": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.0.tgz", - "integrity": "sha512-RUQG3vQ3LX7peqqHAbmayhgrF5aTilPnazinaSGF1P0+tgM3vvIRWPHmlLIz2qFqB9LqFIxditxc8O2Z6psrRw==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/eventstream-serde-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.0.tgz", - "integrity": "sha512-baRPdMBDMBExZXIUAoPGm/hntixjt/VFpU6+VmCyiYJYzRHRxoaI1MN+5XE+hIS8AJ2GCHLMFEIOLzq9xx1EgQ==", - "requires": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/eventstream-serde-universal": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.0.tgz", - "integrity": "sha512-HNFfShmotWGeAoW4ujP8meV9BZavcpmerDbPIjkJbxKbN8RsUcpRQ/2OyIxWNxXNH2GWCAxuSB7ynmIGJlQ3Dw==", - "requires": { - "@smithy/eventstream-codec": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/eventstream-codec": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.0.0.tgz", - "integrity": "sha512-PUtyEA0Oik50SaEFCZ0WPVtF9tz/teze2fDptW6WRXl+RrEenH8UbEjudOz8iakiMl3lE3lCVqYf2Y+znL8QFQ==", - "requires": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/fetch-http-handler": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.2.tgz", - "integrity": "sha512-K7aRtRuaBjzlk+jWWeyfDTLAmRRvmA4fU8eHUXtjsuEDgi3f356ZE32VD2ssxIH13RCLVZbXMt5h7wHzYiSuVA==", - "requires": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/hash-blob-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.0.0.tgz", - "integrity": "sha512-/Wbpdg+bwJvW7lxR/zpWAc1/x/YkcqguuF2bAzkJrvXriZu1vm8r+PUdE4syiVwQg7PPR2dXpi3CLBb9qRDaVQ==", - "requires": { - "@smithy/chunked-blob-reader": "^3.0.0", - "@smithy/chunked-blob-reader-native": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/hash-node": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-2.0.11.tgz", - "integrity": "sha512-PbleVugN2tbhl1ZoNWVrZ1oTFFas/Hq+s6zGO8B9bv4w/StTriTKA9W+xZJACOj9X7zwfoTLbscM+avCB1KqOQ==", - "requires": { - "@smithy/types": "^2.3.5", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/hash-stream-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.0.0.tgz", - "integrity": "sha512-J0i7de+EgXDEGITD4fxzmMX8CyCNETTIRXlxjMiNUvvu76Xn3GJ31wQR85ynlPk2wI1lqoknAFJaD1fiNDlbIA==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/invalid-dependency": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-2.0.11.tgz", - "integrity": "sha512-zazq99ujxYv/NOf9zh7xXbNgzoVLsqE0wle8P/1zU/XdhPi/0zohTPKWUzIxjGdqb5hkkwfBkNkl5H+LE0mvgw==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/is-array-buffer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.0.0.tgz", - "integrity": "sha512-z3PjFjMyZNI98JFRJi/U0nGoLWMSJlDjAW4QUX2WNZLas5C0CmVV6LJ01JI0k90l7FvpmixjWxPFmENSClQ7ug==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/md5-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.0.tgz", - "integrity": "sha512-Tm0vrrVzjlD+6RCQTx7D3Ls58S3FUH1ZCtU1MIh/qQmaOo1H9lMN2as6CikcEwgattnA9SURSdoJJ27xMcEfMA==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/middleware-content-length": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-2.0.13.tgz", - "integrity": "sha512-Md2kxWpaec3bXp1oERFPQPBhOXCkGSAF7uc1E+4rkwjgw3/tqAXRtbjbggu67HJdwaif76As8AV6XxbD1HzqTQ==", - "requires": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/middleware-endpoint": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-2.0.11.tgz", - "integrity": "sha512-mCugsvB15up6fqpzUEpMT4CuJmFkEI+KcozA7QMzYguXCaIilyMKsyxgamwmr+o7lo3QdjN0//XLQ9bWFL129g==", - "requires": { - "@smithy/middleware-serde": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - } - }, - "@smithy/middleware-retry": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-2.0.16.tgz", - "integrity": "sha512-Br5+0yoiMS0ugiOAfJxregzMMGIRCbX4PYo1kDHtLgvkA/d++aHbnHB819m5zOIAMPvPE7AThZgcsoK+WOsUTA==", - "requires": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/protocol-http": "^3.0.7", - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-retry": "^2.0.4", - "tslib": "^2.5.0", - "uuid": "^8.3.2" - }, - "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - } - } - }, - "@smithy/middleware-serde": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-2.0.11.tgz", - "integrity": "sha512-NuxnjMyf4zQqhwwdh0OTj5RqpnuT6HcH5Xg5GrPijPcKzc2REXVEVK4Yyk8ckj8ez1XSj/bCmJ+oNjmqB02GWA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/middleware-stack": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-2.0.5.tgz", - "integrity": "sha512-bVQU/rZzBY7CbSxIrDTGZYnBWKtIw+PL/cRc9B7etZk1IKSOe0NvKMJyWllfhfhrTeMF6eleCzOihIQympAvPw==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/node-config-provider": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-2.1.1.tgz", - "integrity": "sha512-1lF6s1YWBi1LBu2O30tD3jyTgMtuvk/Z1twzXM4GPYe4dmZix4nNREPJIPOcfFikNU2o0eTYP80+izx5F2jIJA==", - "requires": { - "@smithy/property-provider": "^2.0.12", - "@smithy/shared-ini-file-loader": "^2.2.0", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/node-http-handler": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.1.7.tgz", - "integrity": "sha512-PQIKZXlp3awCDn/xNlCSTFE7aYG/5Tx33M05NfQmWYeB5yV1GZZOSz4dXpwiNJYTXb9jPqjl+ueXXkwtEluFFA==", - "requires": { - "@smithy/abort-controller": "^2.0.11", - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/property-provider": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-2.0.12.tgz", - "integrity": "sha512-Un/OvvuQ1Kg8WYtoMCicfsFFuHb/TKL3pCA6ZIo/WvNTJTR94RtoRnL7mY4XkkUAoFMyf6KjcQJ76y1FX7S5rw==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/protocol-http": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-3.0.7.tgz", - "integrity": "sha512-HnZW8y+r66ntYueCDbLqKwWcMNWW8o3eVpSrHNluwtBJ/EUWfQHRKSiu6vZZtc6PGfPQWgVfucoCE/C3QufMAA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/querystring-builder": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-2.0.11.tgz", - "integrity": "sha512-b4kEbVMxpmfv2VWUITn2otckTi7GlMteZQxi+jlwedoATOGEyrCJPfRcYQJjbCi3fZ2QTfh3PcORvB27+j38Yg==", - "requires": { - "@smithy/types": "^2.3.5", - "@smithy/util-uri-escape": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/querystring-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.11.tgz", - "integrity": "sha512-YXe7jhi7s3dQ0Fu9dLoY/gLu6NCyy8tBWJL/v2c9i7/RLpHgKT+uT96/OqZkHizCJ4kr0ZD46tzMjql/o60KLg==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/service-error-classification": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-2.0.4.tgz", - "integrity": "sha512-77506l12I5gxTZqBkx3Wb0RqMG81bMYLaVQ+EqIWFwQDJRs5UFeXogKxSKojCmz1wLUziHZQXm03MBzPQiumQw==", - "requires": { - "@smithy/types": "^2.3.5" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-2.2.0.tgz", - "integrity": "sha512-xFXqs4vAb5BdkzHSRrTapFoaqS4/3m/CGZzdw46fBjYZ0paYuLAoMY60ICCn1FfGirG+PiJ3eWcqJNe4/SkfyA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/signature-v4": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-2.0.11.tgz", - "integrity": "sha512-EFVU1dT+2s8xi227l1A9O27edT/GNKvyAK6lZnIZ0zhIHq/jSLznvkk15aonGAM1kmhmZBVGpI7Tt0odueZK9A==", - "requires": { - "@smithy/eventstream-codec": "^2.0.11", - "@smithy/is-array-buffer": "^2.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-uri-escape": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/smithy-client": { - "version": "2.1.10", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-2.1.10.tgz", - "integrity": "sha512-2OEmZDiW1Z196QHuQZ5M6cBE8FCSG0H2HADP1G+DY8P3agsvb0YJyfhyKuJbxIQy15tr3eDAK6FOrlbxgKOOew==", - "requires": { - "@smithy/middleware-stack": "^2.0.5", - "@smithy/types": "^2.3.5", - "@smithy/util-stream": "^2.0.15", - "tslib": "^2.5.0" - } - }, - "@smithy/types": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.3.5.tgz", - "integrity": "sha512-ehyDt8M9hehyxrLQGoA1BGPou8Js1Ocoh5M0ngDhJMqbFmNK5N6Xhr9/ZExWkyIW8XcGkiMPq3ZUEE0ScrhbuQ==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/url-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.11.tgz", - "integrity": "sha512-h89yXMCCF+S5k9XIoKltMIWTYj+FcEkU/IIFZ6RtE222fskOTL4Iak6ZRG+ehSvZDt8yKEcxqheTDq7JvvtK3g==", - "requires": { - "@smithy/querystring-parser": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-base64": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-2.0.0.tgz", - "integrity": "sha512-Zb1E4xx+m5Lud8bbeYi5FkcMJMnn+1WUnJF3qD7rAdXpaL7UjkFQLdmW5fHadoKbdHpwH9vSR8EyTJFHJs++tA==", - "requires": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-body-length-browser": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz", - "integrity": "sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-body-length-node": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-2.1.0.tgz", - "integrity": "sha512-/li0/kj/y3fQ3vyzn36NTLGmUwAICb7Jbe/CsWCktW363gh1MOcpEcSO3mJ344Gv2dqz8YJCLQpb6hju/0qOWw==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-buffer-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.0.0.tgz", - "integrity": "sha512-/YNnLoHsR+4W4Vf2wL5lGv0ksg8Bmk3GEGxn2vEQt52AQaPSCuaO5PM5VM7lP1K9qHRKHwrPGktqVoAHKWHxzw==", - "requires": { - "@smithy/is-array-buffer": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-config-provider": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-2.0.0.tgz", - "integrity": "sha512-xCQ6UapcIWKxXHEU4Mcs2s7LcFQRiU3XEluM2WcCjjBtQkUN71Tb+ydGmJFPxMUrW/GWMgQEEGipLym4XG0jZg==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-defaults-mode-browser": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-2.0.14.tgz", - "integrity": "sha512-NupG7SWUucm3vJrvlpt9jG1XeoPJphjcivgcUUXhDJbUPy4F04LhlTiAhWSzwlCNcF8OJsMvZ/DWbpYD3pselw==", - "requires": { - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-defaults-mode-node": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-2.0.18.tgz", - "integrity": "sha512-+3jMom/b/Cdp21tDnY4vKu249Al+G/P0HbRbct7/aSZDlROzv1tksaYukon6UUv7uoHn+/McqnsvqZHLlqvQ0g==", - "requires": { - "@smithy/config-resolver": "^2.0.14", - "@smithy/credential-provider-imds": "^2.0.16", - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-endpoints": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.1.tgz", - "integrity": "sha512-ZRT0VCOnKlVohfoABMc8lWeQo/JEFuPWctfNRXgTHbyOVssMOLYFUNWukxxiHRGVAhV+n3c0kPW+zUqckjVPEA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/util-hex-encoding": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz", - "integrity": "sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-middleware": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-2.0.4.tgz", - "integrity": "sha512-Pbu6P4MBwRcjrLgdTR1O4Y3c0sTZn2JdOiJNcgL7EcIStcQodj+6ZTXtbyU/WTEU3MV2NMA10LxFc3AWHZ3+4A==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-retry": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-2.0.4.tgz", - "integrity": "sha512-b+n1jBBKc77C1E/zfBe1Zo7S9OXGBiGn55N0apfhZHxPUP/fMH5AhFUUcWaJh7NAnah284M5lGkBKuhnr3yK5w==", - "requires": { - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-stream": { - "version": "2.0.15", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-2.0.15.tgz", - "integrity": "sha512-A/hkYJPH2N5MCWYvky4tTpQihpYAEzqnUfxDyG3L/yMndy/2sLvxnyQal9Opuj1e9FiKSTeMyjnU9xxZGs0mRw==", - "requires": { - "@smithy/fetch-http-handler": "^2.2.2", - "@smithy/node-http-handler": "^2.1.7", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-uri-escape": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.0.0.tgz", - "integrity": "sha512-ebkxsqinSdEooQduuk9CbKcI+wheijxEb3utGXkCoYQkJnwTnLbH1JXGimJtUkQwNQbsbuYwG2+aFVyZf5TLaw==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-utf8": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.0.0.tgz", - "integrity": "sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ==", - "requires": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-waiter": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.0.0.tgz", - "integrity": "sha512-+fEXJxGDLCoqRKVSmo0auGxaqbiCo+8oph+4auefYjaNxjOLKSY2MxVQfRzo65PaZv4fr+5lWg+au7vSuJJ/zw==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" - }, - "@tsconfig/node10": { - "version": "1.0.9", - "dev": true - }, - "@tsconfig/node12": { - "version": "1.0.11", - "dev": true - }, - "@tsconfig/node14": { - "version": "1.0.3", - "dev": true - }, - "@tsconfig/node16": { - "version": "1.0.3", - "dev": true - }, - "@types/body-parser": { - "version": "1.19.2", - "dev": true, - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" - }, - "@types/chai": { - "version": "4.3.4", - "dev": true - }, - "@types/chai-as-promised": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.6.tgz", - "integrity": "sha512-cQLhk8fFarRVZAXUQV1xEnZgMoPxqKojBvRkqPCKPQCzEhpbbSKl1Uu75kDng7k5Ln6LQLUmNBjLlFthCgm1NA==", - "dev": true, - "requires": { - "@types/chai": "*" - } - }, - "@types/chai-spies": { - "version": "1.0.3", - "dev": true, - "requires": { - "@types/chai": "*" - } - }, - "@types/compression": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.3.tgz", - "integrity": "sha512-rKquEGjebqizyHNMOpaE/4FdYR5VQiWFeesqYfvJU0seSEyB4625UGhNOO/qIkH10S3wftiV7oefc8WdLZ/gCQ==", - "dev": true, - "requires": { - "@types/express": "*" - } - }, - "@types/connect": { - "version": "3.4.35", - "dev": true, - "requires": { - "@types/node": "*" - } - }, - "@types/cookiejar": { - "version": "2.1.2", - "dev": true - }, - "@types/debug": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", - "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", - "requires": { - "@types/ms": "*" - } - }, - "@types/express": { - "version": "4.17.17", - "dev": true, - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.33", - "dev": true, - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/http-errors": { - "version": "2.0.1", - "dev": true - }, - "@types/kafkajs": { - "version": "1.9.0", - "dev": true, - "requires": { - "kafkajs": "*" - } - }, - "@types/knex": { - "version": "0.16.1", - "dev": true, - "requires": { - "knex": "*" - } - }, - "@types/lodash": { - "version": "4.14.191" - }, - "@types/mime": { - "version": "3.0.1", - "dev": true - }, - "@types/mocha": { - "version": "10.0.1", - "dev": true - }, - "@types/mock-knex": { - "version": "0.4.6", - "resolved": "https://registry.npmjs.org/@types/mock-knex/-/mock-knex-0.4.6.tgz", - "integrity": "sha512-7MHM9v9ZgFpAZh2NfCyYeEJKlYJSf/mVRAQ324AJZGB9fbo8dVa8u9nxibyJzwwMXYlbRXLNbKEfoK2xBJFgUw==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, - "@types/moment": { - "version": "2.13.0", - "dev": true, - "requires": { - "moment": "*" - } - }, - "@types/ms": { - "version": "0.7.34", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", - "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" - }, - "@types/node": { - "version": "18.15.3" - }, - "@types/pegjs": { - "version": "0.10.6", - "resolved": "https://registry.npmjs.org/@types/pegjs/-/pegjs-0.10.6.tgz", - "integrity": "sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==" - }, - "@types/pg": { - "version": "8.6.6", - "dev": true, - "requires": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "@types/qs": { - "version": "6.9.7", - "dev": true - }, - "@types/range-parser": { - "version": "1.2.4", - "dev": true - }, - "@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", - "requires": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - }, - "dependencies": { - "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - } - } - }, - "@types/serve-static": { - "version": "1.15.1", - "dev": true, - "requires": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "@types/sinon": { - "version": "17.0.3", - "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.3.tgz", - "integrity": "sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==", - "dev": true, - "requires": { - "@types/sinonjs__fake-timers": "*" - } - }, - "@types/sinonjs__fake-timers": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", - "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", - "dev": true - }, - "@types/slug": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/@types/slug/-/slug-5.0.8.tgz", - "integrity": "sha512-mblTWR1OST257k1gZ3QvqG+ERSr8Ea6dyM1FH6Jtm4jeXi0/r0/95VNctofuiywPxCVQuE8AuFoqmvJ4iVUlXQ==", - "dev": true - }, - "@types/superagent": { - "version": "3.8.7", - "dev": true, - "requires": { - "@types/cookiejar": "*", - "@types/node": "*" - } - }, - "@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" - }, - "@types/triple-beam": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.3.tgz", - "integrity": "sha512-6tOUG+nVHn0cJbVp25JFayS5UE6+xlbcNF9Lo9mU7U0zk3zeUShZied4YEQZjy1JBF043FSkdXw8YkUJuVtB5g==" - }, - "@types/uuid": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz", - "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ==", - "dev": true - }, - "@types/validator": { - "version": "13.11.10", - "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.11.10.tgz", - "integrity": "sha512-e2PNXoXLr6Z+dbfx5zSh9TRlXJrELycxiaXznp4S5+D2M3b9bqJEitNHA5923jhnB2zzFiZHa2f0SI1HoIahpg==" - }, - "@typescript-eslint/eslint-plugin": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.12.0.tgz", - "integrity": "sha512-7F91fcbuDf/d3S8o21+r3ZncGIke/+eWk0EpO21LXhDfLahriZF9CGj4fbAetEjlaBdjdSm9a6VeXbpbT6Z40Q==", - "dev": true, - "requires": { - "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/type-utils": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "graphemer": "^1.4.0", - "ignore": "^5.3.1", - "natural-compare": "^1.4.0", - "ts-api-utils": "^1.3.0" - } - }, - "@typescript-eslint/parser": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.12.0.tgz", - "integrity": "sha512-dm/J2UDY3oV3TKius2OUZIFHsomQmpHtsV0FTh1WO8EKgHLQ1QCADUqscPgTpU+ih1e21FQSRjXckHn3txn6kQ==", - "dev": true, - "requires": { - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@typescript-eslint/scope-manager": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.12.0.tgz", - "integrity": "sha512-itF1pTnN6F3unPak+kutH9raIkL3lhH1YRPGgt7QQOh43DQKVJXmWkpb+vpc/TiDHs6RSd9CTbDsc/Y+Ygq7kg==", - "dev": true, - "requires": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0" - } - }, - "@typescript-eslint/type-utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.12.0.tgz", - "integrity": "sha512-lib96tyRtMhLxwauDWUp/uW3FMhLA6D0rJ8T7HmH7x23Gk1Gwwu8UZ94NMXBvOELn6flSPiBrCKlehkiXyaqwA==", - "dev": true, - "requires": { - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "debug": "^4.3.4", - "ts-api-utils": "^1.3.0" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@typescript-eslint/types": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.12.0.tgz", - "integrity": "sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg==", - "dev": true - }, - "@typescript-eslint/typescript-estree": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.12.0.tgz", - "integrity": "sha512-5bwqLsWBULv1h6pn7cMW5dXX/Y2amRqLaKqsASVwbBHMZSnHqE/HN4vT4fE0aFsiwxYvr98kqOWh1a8ZKXalCQ==", - "dev": true, - "requires": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4", - "globby": "^11.1.0", - "is-glob": "^4.0.3", - "minimatch": "^9.0.4", - "semver": "^7.5.3", - "ts-api-utils": "^1.3.0" - }, - "dependencies": { - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "requires": { - "balanced-match": "^1.0.0" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dev": true, - "requires": { - "brace-expansion": "^2.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@typescript-eslint/utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.12.0.tgz", - "integrity": "sha512-Y6hhwxwDx41HNpjuYswYp6gDbkiZ8Hin9Bf5aJQn1bpTs3afYY4GX+MPYxma8jtoIV2GRwTM/UJm/2uGCVv+DQ==", - "dev": true, - "requires": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0" - } - }, - "@typescript-eslint/visitor-keys": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.12.0.tgz", - "integrity": "sha512-uZk7DevrQLL3vSnfFl5bj4sL75qC9D6EdjemIdbtkuUmIheWpuiiylSY01JxJE7+zGrOWDZrp1WxOuDntvKrHQ==", - "dev": true, - "requires": { - "@typescript-eslint/types": "7.12.0", - "eslint-visitor-keys": "^3.4.3" - } - }, - "@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true - }, - "abbrev": { - "version": "1.1.1", - "dev": true - }, - "abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "requires": { - "event-target-shim": "^5.0.0" - } - }, - "accepts": { - "version": "1.3.8", - "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - } - }, - "acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", - "dev": true - }, - "acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} - }, - "acorn-walk": { - "version": "8.2.0", - "dev": true - }, - "agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "requires": { - "debug": "^4.3.4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "aggregate-error": { - "version": "3.1.0", - "dev": true, - "requires": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - } - }, - "ajv": { - "version": "8.12.0", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "requires": { - "ajv": "^8.0.0" - } - }, - "ansi-colors": { - "version": "4.1.1", - "dev": true - }, - "ansi-regex": { - "version": "5.0.1", - "devOptional": true - }, - "ansi-styles": { - "version": "4.3.0", - "dev": true, - "requires": { - "color-convert": "^2.0.1" - } - }, - "anymatch": { - "version": "3.1.3", - "dev": true, - "requires": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - } - }, - "append-transform": { - "version": "2.0.0", - "dev": true, - "requires": { - "default-require-extensions": "^3.0.0" - } - }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, - "archy": { - "version": "1.0.0", - "dev": true - }, - "are-we-there-yet": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", - "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", - "optional": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "arg": { - "version": "4.1.3", - "dev": true - }, - "argparse": { - "version": "2.0.1", - "dev": true - }, - "array-flatten": { - "version": "1.1.1" - }, - "array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true - }, - "arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" - }, - "assertion-error": { - "version": "1.1.0", - "dev": true - }, - "async": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", - "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", - "requires": { - "lodash": "^4.17.14" - } - }, - "async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "requires": { - "retry": "0.13.1" - } - }, - "asynckit": { - "version": "0.4.0" - }, - "available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" - }, - "aws-sdk": { - "version": "2.1472.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1472.0.tgz", - "integrity": "sha512-U7kAHRbvTy753IXKV8Oom/AqlqnsbXG+Kw5gRbKi6VcsZ3hR/EpNMzdRXTWO5U415bnLWGo8WAqIz67PIaaKsw==", - "requires": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.5.0" - }, - "dependencies": { - "buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==" - }, - "uuid": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", - "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==" - } - } - }, - "axios": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz", - "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==", - "requires": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "balanced-match": { - "version": "1.0.2" - }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, - "big-integer": { - "version": "1.6.52", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", - "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==" - }, - "bignumber.js": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", - "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==" - }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, - "binary-extensions": { - "version": "2.2.0", - "dev": true - }, - "bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "requires": { - "file-uri-to-path": "1.0.0" - } - }, - "bintrees": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", - "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" - }, - "bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", - "requires": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - } - }, - "bowser": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", - "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" - }, - "brace-expansion": { - "version": "1.1.11", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "requires": { - "fill-range": "^7.1.1" - } - }, - "browser-stdout": { - "version": "1.3.1", - "dev": true - }, - "browserslist": { - "version": "4.21.5", - "dev": true, - "requires": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" - } - }, - "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "optional": true, - "requires": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", - "optional": true - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" - }, - "buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" - }, - "buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", - "optional": true - }, - "buffermaker": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz", - "integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==", - "requires": { - "long": "1.1.2" - } - }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==" - }, - "bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" - }, - "caching-transform": { - "version": "4.0.0", - "dev": true, - "requires": { - "hasha": "^5.0.0", - "make-dir": "^3.0.0", - "package-hash": "^4.0.0", - "write-file-atomic": "^3.0.0" - } - }, - "call-bind": { - "version": "1.0.2", - "requires": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - } - }, - "callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true - }, - "camelcase": { - "version": "6.3.0", - "dev": true - }, - "caniuse-lite": { - "version": "1.0.30001466", - "dev": true - }, - "chai": { - "version": "4.3.7", - "dev": true, - "requires": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.2", - "deep-eql": "^4.1.2", - "get-func-name": "^2.0.0", - "loupe": "^2.3.1", - "pathval": "^1.1.1", - "type-detect": "^4.0.5" - } - }, - "chai-as-promised": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", - "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", - "dev": true, - "requires": { - "check-error": "^1.0.2" - } - }, - "chai-http": { - "version": "4.3.0", - "dev": true, - "requires": { - "@types/chai": "4", - "@types/superagent": "^3.8.3", - "cookiejar": "^2.1.1", - "is-ip": "^2.0.0", - "methods": "^1.1.2", - "qs": "^6.5.1", - "superagent": "^3.7.0" - } - }, - "chai-spies": { - "version": "1.0.0", - "dev": true, - "requires": {} - }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, - "chalk": { - "version": "4.1.2", - "dev": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "dependencies": { - "supports-color": { - "version": "7.2.0", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, - "check-error": { - "version": "1.0.2", - "dev": true - }, - "chokidar": { - "version": "3.5.3", - "dev": true, - "requires": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "fsevents": "~2.3.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - } - }, - "chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "optional": true - }, - "clean-stack": { - "version": "2.2.0", - "dev": true - }, - "cliui": { - "version": "7.0.4", - "dev": true, - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", - "optional": true - }, - "color": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", - "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", - "requires": { - "color-convert": "^1.9.3", - "color-string": "^1.6.0" - }, - "dependencies": { - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - } - } - }, - "color-convert": { - "version": "2.0.1", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4" - }, - "color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "requires": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "colorette": { - "version": "2.0.19" - }, - "colors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", - "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==" - }, - "colorspace": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", - "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", - "requires": { - "color": "^3.1.3", - "text-hex": "1.0.x" - } - }, - "combined-stream": { - "version": "1.0.8", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "commander": { - "version": "9.5.0" - }, - "commondir": { - "version": "1.0.1", - "dev": true - }, - "component-emitter": { - "version": "1.3.0", - "dev": true - }, - "compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "requires": { - "mime-db": ">= 1.43.0 < 2" - } - }, - "compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "requires": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "dependencies": { - "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==" - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - } - } - }, - "concat-map": { - "version": "0.0.1" - }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "optional": true - }, - "content-disposition": { - "version": "0.5.4", - "requires": { - "safe-buffer": "5.2.1" - } - }, - "content-type": { - "version": "1.0.5" - }, - "convert-source-map": { - "version": "1.9.0", - "dev": true - }, - "cookie": { - "version": "0.5.0" - }, - "cookie-signature": { - "version": "1.0.6" - }, - "cookiejar": { - "version": "2.1.4", - "dev": true - }, - "core-util-is": { - "version": "1.0.3" - }, - "create-require": { - "version": "1.1.1", - "dev": true - }, - "cross-spawn": { - "version": "7.0.3", - "dev": true, - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, - "cycle": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", - "integrity": "sha512-TVF6svNzeQCOpjCqsy0/CSy8VgObG3wXusJ73xW2GbG5rGx7lC8zxDSURicsXI2UsGdi2L0QNRCi745/wUDvsA==" - }, - "dateformat": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.0.0.tgz", - "integrity": "sha512-k6+FtJ8RoNx9V0yHo6lURQWlFqc8wbo0t/kocHvfMJQiXTW+izR6bubmB9HeWuUiZFtpMrAm+wPMGmGhfPbNlQ==" - }, - "debug": { - "version": "2.6.9", - "requires": { - "ms": "2.0.0" - } - }, - "decamelize": { - "version": "4.0.0", - "dev": true - }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", - "optional": true, - "requires": { - "mimic-response": "^1.0.0" - } - }, - "deep-eql": { - "version": "4.1.3", - "dev": true, - "requires": { - "type-detect": "^4.0.0" - } - }, - "deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "optional": true - }, - "deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, - "default-require-extensions": { - "version": "3.0.1", - "dev": true, - "requires": { - "strip-bom": "^4.0.0" - } - }, - "delayed-stream": { - "version": "1.0.0" - }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "optional": true - }, - "denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" - }, - "depd": { - "version": "2.0.0" - }, - "destroy": { - "version": "1.2.0" - }, - "detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", - "optional": true - }, - "diff": { - "version": "5.0.0", - "dev": true - }, - "diff-json": { - "version": "2.0.0", - "dev": true, - "requires": { - "lodash.difference": ">= 4.0.0", - "lodash.find": ">= 4.0.0", - "lodash.intersection": ">= 4.0.0", - "lodash.keyby": ">= 4.0.0" - } - }, - "dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "requires": { - "path-type": "^4.0.0" - } - }, - "doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "requires": { - "esutils": "^2.0.2" - } - }, - "dottie": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz", - "integrity": "sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==" - }, - "duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "requires": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "ee-first": { - "version": "1.1.1" - }, - "electron-to-chromium": { - "version": "1.4.330", - "dev": true - }, - "emoji-regex": { - "version": "8.0.0", - "devOptional": true - }, - "enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "encodeurl": { - "version": "1.0.2" - }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, - "es6-error": { - "version": "4.1.1", - "dev": true - }, - "escalade": { - "version": "3.1.1" - }, - "escape-html": { - "version": "1.0.3" - }, - "escape-string-regexp": { - "version": "4.0.0", - "dev": true - }, - "eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", - "dev": true, - "requires": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", - "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "@ungap/structured-clone": "^1.2.0", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.3.2", - "doctrine": "^3.0.0", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.2", - "eslint-visitor-keys": "^3.4.3", - "espree": "^9.6.1", - "esquery": "^1.4.2", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "globals": "^13.19.0", - "graphemer": "^1.4.0", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-yaml": "^4.1.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3", - "strip-ansi": "^6.0.1", - "text-table": "^0.2.0" - }, - "dependencies": { - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "requires": { - "is-glob": "^4.0.3" - } - }, - "globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "requires": { - "type-fest": "^0.20.2" - } - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true - } - } - }, - "eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", - "dev": true, - "requires": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - } - }, - "eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true - }, - "esm": { - "version": "3.2.25" - }, - "espree": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", - "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", - "dev": true, - "requires": { - "acorn": "^8.9.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.4.1" - } - }, - "esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", - "dev": true, - "requires": { - "estraverse": "^5.1.0" - } - }, - "esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "requires": { - "estraverse": "^5.2.0" - } - }, - "estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true - }, - "esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true - }, - "etag": { - "version": "1.8.1" - }, - "event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" - }, - "events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" - }, - "expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "optional": true - }, - "express": { - "version": "4.18.2", - "requires": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.1", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.5.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.2.0", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.7", - "qs": "6.11.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "dependencies": { - "body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "requires": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - } - }, - "raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - } - } - }, - "extend": { - "version": "3.0.2" - }, - "eyes": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", - "integrity": "sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==" - }, - "fast-deep-equal": { - "version": "3.1.3" - }, - "fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - } - }, - "fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true - }, - "fast-safe-stringify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", - "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" - }, - "fast-xml-parser": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", - "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", - "requires": { - "strnum": "^1.0.5" - } - }, - "fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", - "dev": true, - "requires": { - "reusify": "^1.0.4" - } - }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "requires": { - "pend": "~1.2.0" - } - }, - "fecha": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", - "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" - }, - "file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", - "dev": true, - "requires": { - "flat-cache": "^3.0.4" - } - }, - "file-stream-rotator": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.2.1.tgz", - "integrity": "sha512-Qf6xva6g0fWqI3LR48eOa8ubF+AP6ftLUSt9Uin6XadP5s3d4vD0J52l/JjmMPiFleCtsVJJ6cyMU2GNRLXQIQ==", - "requires": { - "moment": "^2.11.2" - } - }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, - "fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "requires": { - "to-regex-range": "^5.0.1" - } - }, - "finalhandler": { - "version": "1.2.0", - "requires": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - } - }, - "find-cache-dir": { - "version": "3.3.2", - "dev": true, - "requires": { - "commondir": "^1.0.1", - "make-dir": "^3.0.2", - "pkg-dir": "^4.1.0" - } - }, - "find-up": { - "version": "5.0.0", - "dev": true, - "requires": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - } - }, - "flat": { - "version": "5.0.2", - "dev": true - }, - "flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", - "dev": true, - "requires": { - "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" - } - }, - "flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", - "dev": true - }, - "fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, - "follow-redirects": { - "version": "1.15.2" - }, - "for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "requires": { - "is-callable": "^1.1.3" - } - }, - "foreground-child": { - "version": "2.0.0", - "dev": true, - "requires": { - "cross-spawn": "^7.0.0", - "signal-exit": "^3.0.2" - } - }, - "form-data": { - "version": "4.0.0", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - }, - "formidable": { - "version": "1.2.6", - "dev": true - }, - "forwarded": { - "version": "0.2.0" - }, - "fresh": { - "version": "0.5.2" - }, - "fromentries": { - "version": "1.3.2", - "dev": true - }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "optional": true - }, - "fs.realpath": { - "version": "1.0.0", - "dev": true - }, - "fsevents": { - "version": "2.3.2", - "dev": true, - "optional": true - }, - "function-bind": { - "version": "1.1.1" - }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==", - "optional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "optional": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", - "optional": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", - "optional": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "optional": true, - "requires": { - "ansi-regex": "^2.0.0" - } - } - } - }, - "gaxios": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.6.0.tgz", - "integrity": "sha512-bpOZVQV5gthH/jVCSuYuokRo2bTKOcuBiVWpjmTn6C5Agl5zclGfTljuGsQZxwwDBkli+YhZhP4TdlqTnhOezQ==", - "requires": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "dependencies": { - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "gcp-metadata": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", - "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", - "requires": { - "gaxios": "^6.0.0", - "json-bigint": "^1.0.0" - } - }, - "gensync": { - "version": "1.0.0-beta.2", - "dev": true - }, - "get-caller-file": { - "version": "2.0.5", - "dev": true - }, - "get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true - }, - "get-intrinsic": { - "version": "1.2.0", - "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" - } - }, - "get-package-type": { - "version": "0.1.0" - }, - "getopts": { - "version": "2.3.0" - }, - "github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "optional": true - }, - "glob": { - "version": "7.2.0", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } - } - }, - "glob-parent": { - "version": "5.1.2", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - }, - "globals": { - "version": "11.12.0", - "dev": true - }, - "globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - } - }, - "google-auth-library": { - "version": "9.10.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.10.0.tgz", - "integrity": "sha512-ol+oSa5NbcGdDqA+gZ3G3mev59OHBZksBTxY/tYwjtcp1H/scAFwJfSQU9/1RALoyZ7FslNbke8j4i3ipwlyuQ==", - "requires": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - } - }, - "gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "requires": { - "get-intrinsic": "^1.1.3" - } - }, - "graceful-fs": { - "version": "4.2.10", - "dev": true - }, - "graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true - }, - "gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "requires": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - } - }, - "has": { - "version": "1.0.3", - "requires": { - "function-bind": "^1.1.1" - } - }, - "has-flag": { - "version": "4.0.0", - "dev": true - }, - "has-symbols": { - "version": "1.0.3" - }, - "has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "requires": { - "has-symbols": "^1.0.2" - } - }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "optional": true - }, - "hasha": { - "version": "5.2.2", - "dev": true, - "requires": { - "is-stream": "^2.0.0", - "type-fest": "^0.8.0" - } - }, - "he": { - "version": "1.2.0", - "dev": true - }, - "html-entities": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", - "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==" - }, - "html-escaper": { - "version": "2.0.2", - "dev": true - }, - "http-errors": { - "version": "2.0.0", - "requires": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - } - }, - "http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "requires": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "http-status": { - "version": "1.6.2" - }, - "https-proxy-agent": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", - "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", - "requires": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, - "ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", - "dev": true - }, - "ignore-by-default": { - "version": "1.0.1", - "dev": true - }, - "import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "requires": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "dependencies": { - "resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true - } - } - }, - "imurmurhash": { - "version": "0.1.4", - "dev": true - }, - "indent-string": { - "version": "4.0.0", - "dev": true - }, - "inflection": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", - "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==" - }, - "inflight": { - "version": "1.0.6", - "dev": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4" - }, - "ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "optional": true - }, - "interpret": { - "version": "2.2.0" - }, - "ip-regex": { - "version": "2.1.0", - "dev": true - }, - "ipaddr.js": { - "version": "1.9.1" - }, - "is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "requires": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - } - }, - "is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, - "is-binary-path": { - "version": "2.1.0", - "dev": true, - "requires": { - "binary-extensions": "^2.0.0" - } - }, - "is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" - }, - "is-core-module": { - "version": "2.11.0", - "requires": { - "has": "^1.0.3" - } - }, - "is-extglob": { - "version": "2.1.1", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "devOptional": true - }, - "is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "requires": { - "has-tostringtag": "^1.0.0" - } - }, - "is-glob": { - "version": "4.0.3", - "dev": true, - "requires": { - "is-extglob": "^2.1.1" - } - }, - "is-ip": { - "version": "2.0.0", - "dev": true, - "requires": { - "ip-regex": "^2.0.0" - } - }, - "is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true - }, - "is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true - }, - "is-plain-obj": { - "version": "2.1.0", - "dev": true - }, - "is-stream": { - "version": "2.0.1" - }, - "is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "requires": { - "which-typed-array": "^1.1.11" - } - }, - "is-typedarray": { - "version": "1.0.0", - "dev": true - }, - "is-unicode-supported": { - "version": "0.1.0", - "dev": true - }, - "is-windows": { - "version": "1.0.2", - "dev": true - }, - "isarray": { - "version": "1.0.0" - }, - "isexe": { - "version": "2.0.0", - "dev": true - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" - }, - "istanbul-lib-coverage": { - "version": "3.2.0", - "dev": true - }, - "istanbul-lib-hook": { - "version": "3.0.0", - "dev": true, - "requires": { - "append-transform": "^2.0.0" - } - }, - "istanbul-lib-instrument": { - "version": "4.0.3", - "dev": true, - "requires": { - "@babel/core": "^7.7.5", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.0.0", - "semver": "^7.5.3" - } - }, - "istanbul-lib-processinfo": { - "version": "2.0.3", - "dev": true, - "requires": { - "archy": "^1.0.0", - "cross-spawn": "^7.0.3", - "istanbul-lib-coverage": "^3.2.0", - "p-map": "^3.0.0", - "rimraf": "^3.0.0", - "uuid": "^8.3.2" - }, - "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true - } - } - }, - "istanbul-lib-report": { - "version": "3.0.0", - "dev": true, - "requires": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^3.0.0", - "supports-color": "^7.1.0" - }, - "dependencies": { - "supports-color": { - "version": "7.2.0", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, - "istanbul-lib-source-maps": { - "version": "4.0.1", - "dev": true, - "requires": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "istanbul-reports": { - "version": "3.1.5", - "dev": true, - "requires": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - } - }, - "jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" - }, - "js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true - }, - "js-yaml": { - "version": "4.1.0", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - }, - "jsesc": { - "version": "2.5.2", - "dev": true - }, - "json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "requires": { - "bignumber.js": "^9.0.0" - } - }, - "json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true - }, - "json-schema-traverse": { - "version": "1.0.0" - }, - "json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true - }, - "json-stringify-safe": { - "version": "5.0.1" - }, - "json5": { - "version": "2.2.3", - "dev": true - }, - "just-extend": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", - "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", - "dev": true - }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "kafka-node": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-5.0.0.tgz", - "integrity": "sha512-dD2ga5gLcQhsq1yNoQdy1MU4x4z7YnXM5bcG9SdQuiNr5KKuAmXixH1Mggwdah5o7EfholFbcNDPSVA6BIfaug==", - "requires": { - "async": "^2.6.2", - "binary": "~0.3.0", - "bl": "^2.2.0", - "buffer-crc32": "~0.2.5", - "buffermaker": "~1.2.0", - "debug": "^2.1.3", - "denque": "^1.3.0", - "lodash": "^4.17.4", - "minimatch": "^3.0.2", - "nested-error-stacks": "^2.0.0", - "optional": "^0.1.3", - "retry": "^0.10.1", - "snappy": "^6.0.1", - "uuid": "^3.0.0" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "retry": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", - "integrity": "sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==" - } - } - }, - "kafkajs": { - "version": "2.2.4" - }, - "kafkajs-snappy": { - "version": "1.1.0", - "requires": { - "snappyjs": "^0.6.0" - } - }, - "kafkajs-snappy-typescript": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/kafkajs-snappy-typescript/-/kafkajs-snappy-typescript-1.0.3.tgz", - "integrity": "sha512-H5CiRKQ+RGJprWixvY5gDF3ofZEqQA0FFo0ePNgDCoJuH87n3gGUogtlUuei6A8c1ad/5qsQS3f9SSnmB/me4A==", - "requires": { - "snappy": "^7.2.2" - }, - "dependencies": { - "snappy": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.2.2.tgz", - "integrity": "sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==", - "requires": { - "@napi-rs/snappy-android-arm-eabi": "7.2.2", - "@napi-rs/snappy-android-arm64": "7.2.2", - "@napi-rs/snappy-darwin-arm64": "7.2.2", - "@napi-rs/snappy-darwin-x64": "7.2.2", - "@napi-rs/snappy-freebsd-x64": "7.2.2", - "@napi-rs/snappy-linux-arm-gnueabihf": "7.2.2", - "@napi-rs/snappy-linux-arm64-gnu": "7.2.2", - "@napi-rs/snappy-linux-arm64-musl": "7.2.2", - "@napi-rs/snappy-linux-x64-gnu": "7.2.2", - "@napi-rs/snappy-linux-x64-musl": "7.2.2", - "@napi-rs/snappy-win32-arm64-msvc": "7.2.2", - "@napi-rs/snappy-win32-ia32-msvc": "7.2.2", - "@napi-rs/snappy-win32-x64-msvc": "7.2.2" - } - } - } - }, - "keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "requires": { - "json-buffer": "3.0.1" - } - }, - "knex": { - "version": "2.4.2", - "requires": { - "colorette": "2.0.19", - "commander": "^9.1.0", - "debug": "4.3.4", - "escalade": "^3.1.1", - "esm": "^3.2.25", - "get-package-type": "^0.1.0", - "getopts": "2.3.0", - "interpret": "^2.2.0", - "lodash": "^4.17.21", - "pg-connection-string": "2.5.0", - "rechoir": "^0.8.0", - "resolve-from": "^5.0.0", - "tarn": "^3.0.2", - "tildify": "2.0.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2" - } - } - }, - "kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, - "levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "requires": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - } - }, - "locate-path": { - "version": "6.0.0", - "dev": true, - "requires": { - "p-locate": "^5.0.0" - } - }, - "lodash": { - "version": "4.17.21" - }, - "lodash.difference": { - "version": "4.5.0", - "dev": true - }, - "lodash.find": { - "version": "4.6.0", - "dev": true - }, - "lodash.flattendeep": { - "version": "4.4.0", - "dev": true - }, - "lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", - "dev": true - }, - "lodash.intersection": { - "version": "4.4.0", - "dev": true - }, - "lodash.keyby": { - "version": "4.6.0", - "dev": true - }, - "lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true - }, - "log-symbols": { - "version": "4.1.0", - "dev": true, - "requires": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - } - }, - "logform": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz", - "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==", - "requires": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^2.3.3", - "ms": "^2.1.1", - "triple-beam": "^1.2.0" - }, - "dependencies": { - "colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - }, - "long": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz", - "integrity": "sha512-pjR3OP1X2VVQhCQlrq3s8UxugQsuoucwMOn9Yj/kN/61HMc+lDFJS5bvpNEHneZ9NVaSm8gNWxZvtGS7lqHb3Q==" - }, - "loupe": { - "version": "2.3.6", - "dev": true, - "requires": { - "get-func-name": "^2.0.0" - } - }, - "lru-cache": { - "version": "5.1.1", - "dev": true, - "requires": { - "yallist": "^3.0.2" - } - }, - "make-dir": { - "version": "3.1.0", - "dev": true, - "requires": { - "semver": "^7.5.3" - } - }, - "make-error": { - "version": "1.3.6", - "dev": true - }, - "media-typer": { - "version": "0.3.0" - }, - "merge-descriptors": { - "version": "1.0.1" - }, - "merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true - }, - "methods": { - "version": "1.1.2" - }, - "micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", - "dev": true, - "requires": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - } - }, - "mime": { - "version": "1.6.0" - }, - "mime-db": { - "version": "1.52.0" - }, - "mime-types": { - "version": "2.1.35", - "requires": { - "mime-db": "1.52.0" - } - }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "optional": true - }, - "minimatch": { - "version": "5.0.1", - "dev": true, - "requires": { - "brace-expansion": "^2.0.1" - }, - "dependencies": { - "brace-expansion": { - "version": "2.0.1", - "dev": true, - "requires": { - "balanced-match": "^1.0.0" - } - } - } - }, - "minimist": { - "version": "1.2.8", - "devOptional": true - }, - "mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "optional": true, - "requires": { - "minimist": "^1.2.6" - } - }, - "mocha": { - "version": "10.2.0", - "dev": true, - "requires": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - }, - "dependencies": { - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "ms": { - "version": "2.1.3", - "dev": true - } - } - }, - "moment": { - "version": "2.29.4" - }, - "moment-timezone": { - "version": "0.5.45", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", - "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", - "requires": { - "moment": "^2.29.4" - } - }, - "ms": { - "version": "2.0.0" - }, - "multiparty": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/multiparty/-/multiparty-4.2.1.tgz", - "integrity": "sha512-AvESCnNoQlZiOfP9R4mxN8M9csy2L16EIbWIkt3l4FuGti9kXBS8QVzlfyg4HEnarJhrzZilgNFlZtqmoiAIIA==", - "requires": { - "fd-slicer": "1.1.0", - "http-errors": "~1.7.0", - "safe-buffer": "5.1.2", - "uid-safe": "2.1.5" - }, - "dependencies": { - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" - }, - "http-errors": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz", - "integrity": "sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" - }, - "toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" - } - } - }, - "nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", - "optional": true - }, - "nanoid": { - "version": "3.3.3", - "dev": true - }, - "napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "optional": true - }, - "natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "negotiator": { - "version": "0.6.3" - }, - "nested-error-stacks": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.1.tgz", - "integrity": "sha512-9iN1ka/9zmX1ZvLV9ewJYEk9h7RyRRtqdK0woXcqohu8EWIerfPUjYJPg0ULy0UqP7cslmdGc8xKDJcojlKiaw==" - }, - "nise": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.9.tgz", - "integrity": "sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/text-encoding": "^0.7.2", - "just-extend": "^6.2.0", - "path-to-regexp": "^6.2.1" - }, - "dependencies": { - "path-to-regexp": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", - "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==", - "dev": true - } - } - }, - "nock": { - "version": "13.3.0", - "dev": true, - "requires": { - "debug": "^4.1.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.21", - "propagate": "^2.0.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "node-abi": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", - "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==", - "optional": true, - "requires": { - "semver": "^7.5.3" - } - }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, - "node-preload": { - "version": "0.2.1", - "dev": true, - "requires": { - "process-on-spawn": "^1.0.0" - } - }, - "node-releases": { - "version": "2.0.10", - "dev": true - }, - "node-sql-parser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/node-sql-parser/-/node-sql-parser-5.2.0.tgz", - "integrity": "sha512-lO/9ox0fLl47ksqlwGrorXm8BD+5UbxvUhj6T1XiCXnvoVwWWn7gC2l396p9OXNii19dc4MkyTN3XeJGNUZ8jw==", - "requires": { - "@types/pegjs": "^0.10.0", - "big-integer": "^1.6.48" - } - }, - "nodemon": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.1.tgz", - "integrity": "sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==", - "dev": true, - "requires": { - "chokidar": "^3.5.2", - "debug": "^3.2.7", - "ignore-by-default": "^1.0.1", - "minimatch": "^3.1.2", - "pstree.remy": "^1.1.8", - "semver": "^7.5.3", - "simple-update-notifier": "^2.0.0", - "supports-color": "^5.5.0", - "touch": "^3.1.0", - "undefsafe": "^2.0.5" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "has-flag": { - "version": "3.0.0", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.3", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha512-6kM8CLXvuW5crTxsAtva2YLrRrDaiTIkIePWs9moLHqbFWT94WpNFjwS/5dfLfECg5i/lkmw3aoqVidxt23TEQ==", - "optional": true - }, - "nopt": { - "version": "1.0.10", - "dev": true, - "requires": { - "abbrev": "1" - } - }, - "normalize-path": { - "version": "3.0.0", - "dev": true - }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "optional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==", - "optional": true - }, - "nyc": { - "version": "15.1.0", - "dev": true, - "requires": { - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "caching-transform": "^4.0.0", - "convert-source-map": "^1.7.0", - "decamelize": "^1.2.0", - "find-cache-dir": "^3.2.0", - "find-up": "^4.1.0", - "foreground-child": "^2.0.0", - "get-package-type": "^0.1.0", - "glob": "^7.1.6", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-hook": "^3.0.0", - "istanbul-lib-instrument": "^4.0.0", - "istanbul-lib-processinfo": "^2.0.2", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.0.2", - "make-dir": "^3.0.0", - "node-preload": "^0.2.1", - "p-map": "^3.0.0", - "process-on-spawn": "^1.0.0", - "resolve-from": "^5.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "spawn-wrap": "^2.0.0", - "test-exclude": "^6.0.0", - "yargs": "^15.0.2" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "dev": true - }, - "cliui": { - "version": "6.0.0", - "dev": true, - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "decamelize": { - "version": "1.2.0", - "dev": true - }, - "find-up": { - "version": "4.1.0", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-limit": { - "version": "2.3.0", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "wrap-ansi": { - "version": "6.2.0", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "y18n": { - "version": "4.0.3", - "dev": true - }, - "yargs": { - "version": "15.4.1", - "dev": true, - "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - } - }, - "yargs-parser": { - "version": "18.1.3", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } - } - } - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "optional": true - }, - "object-hash": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", - "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==" - }, - "object-inspect": { - "version": "1.12.3" - }, - "on-finished": { - "version": "2.4.1", - "requires": { - "ee-first": "1.1.1" - } - }, - "on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==" - }, - "once": { - "version": "1.4.0", - "requires": { - "wrappy": "1" - } - }, - "one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "requires": { - "fn.name": "1.x.x" - } - }, - "optional": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", - "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" - }, - "optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "requires": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - } - }, - "os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", - "optional": true - }, - "p-limit": { - "version": "3.1.0", - "requires": { - "yocto-queue": "^0.1.0" - } - }, - "p-locate": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-limit": "^3.0.2" - } - }, - "p-map": { - "version": "3.0.0", - "dev": true, - "requires": { - "aggregate-error": "^3.0.0" - } - }, - "p-try": { - "version": "2.2.0", - "dev": true - }, - "package-hash": { - "version": "4.0.0", - "dev": true, - "requires": { - "graceful-fs": "^4.1.15", - "hasha": "^5.0.0", - "lodash.flattendeep": "^4.4.0", - "release-zalgo": "^1.0.0" - } - }, - "parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "requires": { - "callsites": "^3.0.0" - } - }, - "parseurl": { - "version": "1.3.3" - }, - "path-exists": { - "version": "4.0.0", - "dev": true - }, - "path-is-absolute": { - "version": "1.0.1", - "dev": true - }, - "path-key": { - "version": "3.1.1", - "dev": true - }, - "path-parse": { - "version": "1.0.7" - }, - "path-to-regexp": { - "version": "0.1.7" - }, - "path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true - }, - "pathval": { - "version": "1.1.1", - "dev": true - }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" - }, - "pg": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", - "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", - "requires": { - "pg-cloudflare": "^1.1.1", - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "dependencies": { - "pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - } - } - }, - "pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "pg-connection-string": { - "version": "2.5.0" - }, - "pg-hstore": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", - "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", - "requires": { - "underscore": "^1.13.1" - } - }, - "pg-int8": { - "version": "1.0.1" - }, - "pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "requires": {} - }, - "pg-protocol": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", - "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" - }, - "pg-types": { - "version": "2.2.0", - "requires": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - } - }, - "pgpass": { - "version": "1.0.5", - "requires": { - "split2": "^4.1.0" - } - }, - "picocolors": { - "version": "1.0.0", - "dev": true - }, - "picomatch": { - "version": "2.3.1", - "dev": true - }, - "pkg-dir": { - "version": "4.2.0", - "dev": true, - "requires": { - "find-up": "^4.0.0" - }, - "dependencies": { - "find-up": { - "version": "4.1.0", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-limit": { - "version": "2.3.0", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - } - } - }, - "postgres-array": { - "version": "2.0.0" - }, - "postgres-bytea": { - "version": "1.0.0" - }, - "postgres-date": { - "version": "1.0.7" - }, - "postgres-interval": { - "version": "1.2.0", - "requires": { - "xtend": "^4.0.0" - } - }, - "prebuild-install": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", - "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", - "optional": true, - "requires": { - "detect-libc": "^1.0.3", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "napi-build-utils": "^1.0.1", - "node-abi": "^2.7.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.2.7", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - } - }, - "prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true - }, - "process-nextick-args": { - "version": "2.0.1" - }, - "process-on-spawn": { - "version": "1.0.0", - "dev": true, - "requires": { - "fromentries": "^1.2.0" - } - }, - "prom-client": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.2.0.tgz", - "integrity": "sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA==", - "requires": { - "tdigest": "^0.1.1" - } - }, - "propagate": { - "version": "2.0.1", - "dev": true - }, - "proxy-addr": { - "version": "2.0.7", - "requires": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - } - }, - "proxy-from-env": { - "version": "1.1.0" - }, - "pstree.remy": { - "version": "1.1.8", - "dev": true - }, - "pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "optional": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "punycode": { - "version": "2.3.0" - }, - "qs": { - "version": "6.11.0", - "requires": { - "side-channel": "^1.0.4" - } - }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true - }, - "random-bytes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==" - }, - "randombytes": { - "version": "2.1.0", - "dev": true, - "requires": { - "safe-buffer": "^5.1.0" - } - }, - "range-parser": { - "version": "1.2.1" - }, - "raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - }, - "rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "optional": true, - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "optional": true - } - } - }, - "readable-stream": { - "version": "2.3.8", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - }, - "dependencies": { - "safe-buffer": { - "version": "5.1.2" - } - } - }, - "readdirp": { - "version": "3.6.0", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } - }, - "rechoir": { - "version": "0.8.0", - "requires": { - "resolve": "^1.20.0" - } - }, - "release-zalgo": { - "version": "1.0.0", - "dev": true, - "requires": { - "es6-error": "^4.0.1" - } - }, - "require-directory": { - "version": "2.1.1", - "dev": true - }, - "require-from-string": { - "version": "2.0.2" - }, - "require-main-filename": { - "version": "2.0.0", - "dev": true - }, - "resolve": { - "version": "1.22.1", - "requires": { - "is-core-module": "^2.9.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - } - }, - "resolve-from": { - "version": "5.0.0" - }, - "retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" - }, - "retry-as-promised": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-7.0.4.tgz", - "integrity": "sha512-XgmCoxKWkDofwH8WddD0w85ZfqYz+ZHlr5yo+3YUCfycWawU56T5ckWXsScsj5B8tqUcIG67DxXByo3VUgiAdA==" - }, - "retry-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", - "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", - "requires": { - "@types/request": "^2.48.8", - "extend": "^3.0.2", - "teeny-request": "^9.0.0" - } - }, - "reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true - }, - "rimraf": { - "version": "3.0.2", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - }, - "run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "requires": { - "queue-microtask": "^1.2.2" - } - }, - "safe-buffer": { - "version": "5.2.1" - }, - "safe-stable-stringify": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", - "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" - }, - "semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "requires": { - "lru-cache": "^6.0.0" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - } - }, - "send": { - "version": "0.18.0", - "requires": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "dependencies": { - "ms": { - "version": "2.1.3" - } - } - }, - "sequelize": { - "version": "6.37.3", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.3.tgz", - "integrity": "sha512-V2FTqYpdZjPy3VQrZvjTPnOoLm0KudCRXfGWp48QwhyPPp2yW8z0p0sCYZd/em847Tl2dVxJJ1DR+hF+O77T7A==", - "requires": { - "@types/debug": "^4.1.8", - "@types/validator": "^13.7.17", - "debug": "^4.3.4", - "dottie": "^2.0.6", - "inflection": "^1.13.4", - "lodash": "^4.17.21", - "moment": "^2.29.4", - "moment-timezone": "^0.5.43", - "pg-connection-string": "^2.6.1", - "retry-as-promised": "^7.0.4", - "semver": "^7.5.3", - "sequelize-pool": "^7.1.0", - "toposort-class": "^1.0.1", - "uuid": "^8.3.2", - "validator": "^13.9.0", - "wkx": "^0.5.0" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - } - } - }, - "sequelize-pool": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz", - "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg==" - }, - "serialize-javascript": { - "version": "6.0.0", - "dev": true, - "requires": { - "randombytes": "^2.1.0" - } - }, - "serve-static": { - "version": "1.15.0", - "requires": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.18.0" - } - }, - "set-blocking": { - "version": "2.0.0", - "devOptional": true - }, - "setprototypeof": { - "version": "1.2.0" - }, - "shebang-command": { - "version": "2.0.0", - "dev": true, - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "dev": true - }, - "side-channel": { - "version": "1.0.4", - "requires": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" - } - }, - "signal-exit": { - "version": "3.0.7", - "devOptional": true - }, - "simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "optional": true - }, - "simple-get": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", - "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", - "optional": true, - "requires": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "requires": { - "is-arrayish": "^0.3.1" - } - }, - "simple-update-notifier": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", - "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", - "dev": true, - "requires": { - "semver": "^7.5.3" - } - }, - "sinon": { - "version": "17.0.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", - "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.5", - "supports-color": "^7.2.0" - }, - "dependencies": { - "diff": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", - "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", - "dev": true - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, - "slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true - }, - "slug": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/slug/-/slug-9.1.0.tgz", - "integrity": "sha512-ioOsCfzQSu+D6NZ8XMCR8IW9FgvF8W7Xzz56hBkB/ALvNaWeBs2MUvvPugq3GCrxfHPFeK6hAxGkY/WLnfX2Lg==" - }, - "snappy": { - "version": "6.3.5", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", - "integrity": "sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA==", - "optional": true, - "requires": { - "bindings": "^1.3.1", - "nan": "^2.14.1", - "prebuild-install": "5.3.0" - } - }, - "snappyjs": { - "version": "0.6.1" - }, - "source-map": { - "version": "0.6.1", - "dev": true - }, - "spawn-wrap": { - "version": "2.0.0", - "dev": true, - "requires": { - "foreground-child": "^2.0.0", - "is-windows": "^1.0.2", - "make-dir": "^3.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "which": "^2.0.1" - } - }, - "split2": { - "version": "4.1.0" - }, - "stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==" - }, - "statuses": { - "version": "2.0.1" - }, - "stream-browserify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", - "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", - "requires": { - "inherits": "~2.0.4", - "readable-stream": "^3.5.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "requires": { - "stubs": "^3.0.0" - } - }, - "stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" - }, - "string_decoder": { - "version": "1.1.1", - "requires": { - "safe-buffer": "~5.1.0" - }, - "dependencies": { - "safe-buffer": { - "version": "5.1.2" - } - } - }, - "string-width": { - "version": "4.2.3", - "devOptional": true, - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "devOptional": true, - "requires": { - "ansi-regex": "^5.0.1" - } - }, - "strip-bom": { - "version": "4.0.0", - "dev": true - }, - "strip-json-comments": { - "version": "3.1.1", - "dev": true - }, - "strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, - "stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" - }, - "superagent": { - "version": "3.8.3", - "dev": true, - "requires": { - "component-emitter": "^1.2.0", - "cookiejar": "^2.1.0", - "debug": "^3.1.0", - "extend": "^3.0.0", - "form-data": "^2.3.1", - "formidable": "^1.2.0", - "methods": "^1.1.1", - "mime": "^1.4.1", - "qs": "^6.5.1", - "readable-stream": "^2.3.5" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "form-data": { - "version": "2.5.1", - "dev": true, - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "ms": { - "version": "2.1.3", - "dev": true - } - } - }, - "supports-color": { - "version": "8.1.1", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - }, - "supports-preserve-symlinks-flag": { - "version": "1.0.0" - }, - "tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "optional": true, - "requires": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - }, - "dependencies": { - "pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "optional": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - } - } - }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "optional": true, - "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "dependencies": { - "bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", - "optional": true, - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - } - } - }, - "tarn": { - "version": "3.0.2" - }, - "tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", - "requires": { - "bintrees": "1.0.2" - } - }, - "teeny-request": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", - "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", - "requires": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.9", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "dependencies": { - "agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "requires": { - "debug": "4" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "requires": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - } - }, - "https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "requires": { - "agent-base": "6", - "debug": "4" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "test-exclude": { - "version": "6.0.0", - "dev": true, - "requires": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } - } - }, - "text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, - "text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true - }, - "tildify": { - "version": "2.0.0" - }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", - "optional": true - }, - "to-fast-properties": { - "version": "2.0.0", - "dev": true - }, - "to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "requires": { - "is-number": "^7.0.0" - } - }, - "toidentifier": { - "version": "1.0.1" - }, - "toposort-class": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", - "integrity": "sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg==" - }, - "touch": { - "version": "3.1.0", - "dev": true, - "requires": { - "nopt": "~1.0.10" - } - }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==" - }, - "trino-client": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/trino-client/-/trino-client-0.2.2.tgz", - "integrity": "sha512-TMndAbFiiGlAbJotsqsbMSqFZ8sPtE+KBIl/qYEAI9+eAXruNvz0BhxFC67PE39eouVn+JtF6L8Yr2q6/sAlAA==", - "requires": { - "axios": "1.6.2" - } - }, - "triple-beam": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", - "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==" - }, - "ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", - "dev": true, - "requires": {} - }, - "ts-node": { - "version": "10.9.1", - "dev": true, - "requires": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "dependencies": { - "diff": { - "version": "4.0.2", - "dev": true - } - } - }, - "tsconfig-paths": { - "version": "4.1.2", - "dev": true, - "requires": { - "json5": "^2.2.2", - "minimist": "^1.2.6", - "strip-bom": "^3.0.0" - }, - "dependencies": { - "strip-bom": { - "version": "3.0.0", - "dev": true - } - } - }, - "tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "optional": true, - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "requires": { - "prelude-ls": "^1.2.1" - } - }, - "type-detect": { - "version": "4.0.8", - "dev": true - }, - "type-fest": { - "version": "0.8.1", - "dev": true - }, - "type-is": { - "version": "1.6.18", - "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - } - }, - "typedarray-to-buffer": { - "version": "3.1.5", - "dev": true, - "requires": { - "is-typedarray": "^1.0.0" - } - }, - "typescript": { - "version": "4.9.5", - "dev": true - }, - "uid-safe": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", - "requires": { - "random-bytes": "~1.0.0" - } - }, - "undefsafe": { - "version": "2.0.5", - "dev": true - }, - "underscore": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", - "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" - }, - "unpipe": { - "version": "1.0.0" - }, - "update-browserslist-db": { - "version": "1.0.10", - "dev": true, - "requires": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" - } - }, - "uri-js": { - "version": "4.4.1", - "requires": { - "punycode": "^2.1.0" - } - }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - }, - "dependencies": { - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" - } - } - }, - "util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "requires": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, - "util-deprecate": { - "version": "1.0.2" - }, - "utils-merge": { - "version": "1.0.1" - }, - "uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" - }, - "v8-compile-cache-lib": { - "version": "3.0.1", - "dev": true - }, - "validator": { - "version": "13.12.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.12.0.tgz", - "integrity": "sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==" - }, - "vary": { - "version": "1.1.2" - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "which": { - "version": "2.0.2", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, - "which-module": { - "version": "2.0.0", - "dev": true - }, - "which-pm-runs": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.1.0.tgz", - "integrity": "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==", - "optional": true - }, - "which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "requires": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - } - }, - "wide-align": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", - "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "optional": true, - "requires": { - "string-width": "^1.0.2 || 2 || 3 || 4" - } - }, - "winston": { - "version": "2.4.7", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz", - "integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==", - "requires": { - "async": "^2.6.4", - "colors": "1.0.x", - "cycle": "1.0.x", - "eyes": "0.1.x", - "isstream": "0.1.x", - "stack-trace": "0.0.x" - } - }, - "winston-compat": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/winston-compat/-/winston-compat-0.1.5.tgz", - "integrity": "sha512-EPvPcHT604AV3Ji6d3+vX8ENKIml9VSxMRnPQ+cuK/FX6f3hvPP2hxyoeeCOCFvDrJEujalfcKWlWPvAnFyS9g==", - "requires": { - "cycle": "~1.0.3", - "logform": "^1.6.0", - "triple-beam": "^1.2.0" - } - }, - "winston-daily-rotate-file": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-3.2.3.tgz", - "integrity": "sha512-BOvmvQH2WaiexOjzj14YNHSc18IDyZJ9t4pMsbTERjpjMltoBVijM8DDJJPr2jSqELSNnbgGPBk3kDQSRgOAtQ==", - "requires": { - "file-stream-rotator": "^0.2.1", - "semver": "^7.5.3", - "triple-beam": "^1.3.0", - "winston-compat": "^0.1.4", - "winston-transport": "^4.2.0" - } - }, - "winston-transport": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.0.tgz", - "integrity": "sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==", - "requires": { - "logform": "^2.3.2", - "readable-stream": "^3.6.0", - "triple-beam": "^1.3.0" - }, - "dependencies": { - "@colors/colors": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", - "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" - }, - "fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "logform": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.5.1.tgz", - "integrity": "sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==", - "requires": { - "@colors/colors": "1.5.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "wkx": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.5.0.tgz", - "integrity": "sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg==", - "requires": { - "@types/node": "*" - } - }, - "word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true - }, - "workerpool": { - "version": "6.2.1", - "dev": true - }, - "wrap-ansi": { - "version": "7.0.0", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "wrappy": { - "version": "1.0.2" - }, - "write-file-atomic": { - "version": "3.0.3", - "dev": true, - "requires": { - "imurmurhash": "^0.1.4", - "is-typedarray": "^1.0.0", - "signal-exit": "^3.0.2", - "typedarray-to-buffer": "^3.1.5" - } - }, - "xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - } - }, - "xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" - }, - "xtend": { - "version": "4.0.2" - }, - "y18n": { - "version": "5.0.8", - "dev": true - }, - "yallist": { - "version": "3.1.1", - "dev": true - }, - "yargs": { - "version": "16.2.0", - "dev": true, - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - }, - "yargs-parser": { - "version": "20.2.4", - "dev": true - }, - "yargs-unparser": { - "version": "2.0.0", - "dev": true, - "requires": { - "camelcase": "^6.0.0", - "decamelize": "^4.0.0", - "flat": "^5.0.2", - "is-plain-obj": "^2.1.0" - } - }, - "yn": { - "version": "3.1.1", - "dev": true - }, - "yocto-queue": { - "version": "0.1.0" - } - } -} diff --git a/api-service/package.json b/api-service/package.json index 04b8a85f..acde61cc 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -5,9 +5,8 @@ "main": "dist/app.js", "scripts": { "start": "ts-node ./src/app.ts", - "test": "source .env.test && nyc mocha ./src/v1/test/*.spec.ts --exit && nyc mocha ./src/v2/tests/**/*.spec.ts --exit", - "actions:test": "nyc mocha ./src/v1/test/*.spec.ts --exit", - "actions:test:v2": "nyc mocha ./src/v2/tests/**/*.spec.ts --exit", + "test": "source .env.test && nyc mocha ./src/tests/**/*.spec.ts --exit", + "actions:test": "nyc mocha ./src/tests/**/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", "package": "npm run build && cd dist && npm pack . && cd ..", "lint": "eslint . --ext .ts", @@ -22,6 +21,7 @@ "@aws-sdk/s3-request-presigner": "^3.540.0", "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", + "@jsonhero/schema-infer": "^0.1.5", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", @@ -30,24 +30,27 @@ "body-parser": "^1.20.2", "compression": "^1.7.4", "dateformat": "2.0.0", - "express": "^4.18.2", + "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", + "jsonwebtoken": "^9.0.1", "kafka-node": "^5.0.0", "kafkajs": "^2.2.4", "kafkajs-snappy": "^1.1.0", "kafkajs-snappy-typescript": "^1.0.3", - "sequelize": "^6.37.1", "knex": "^2.4.2", "lodash": "^4.17.21", + "log4js": "^6.9.1", "moment": "^2.29.4", "multiparty": "4.2.1", "node-sql-parser": "^5.1.0", "pg": "^8.11.3", "pg-hstore": "^2.3.4", "prom-client": "^14.2.0", - "trino-client": "^0.2.2", + "redis": "^4.6.15", + "sequelize": "^6.37.1", "slug": "^9.0.0", + "trino-client": "^0.2.2", "uuid": "3.1.0", "winston": "~2.4.3", "winston-daily-rotate-file": "~3.2.1" @@ -57,31 +60,32 @@ "@babel/traverse": "7.23.2" }, "devDependencies": { - "eslint": "^8.57.0", - "@typescript-eslint/eslint-plugin": "^7.1.1", - "@typescript-eslint/parser": "^7.1.1", "@types/chai": "^4.3.3", "@types/chai-as-promised": "^7.1.5", "@types/chai-spies": "^1.0.3", "@types/compression": "^1.7.2", "@types/express": "^4.17.14", "@types/http-errors": "^2.0.1", + "@types/jsonwebtoken": "^9.0.6", "@types/kafkajs": "^1.9.0", "@types/knex": "^0.16.1", "@types/lodash": "^4.14.190", "@types/mocha": "^10.0.0", "@types/mock-knex": "^0.4.4", "@types/moment": "^2.13.0", - "@types/node": "^18.11.9", + "@types/node": "^18.19.39", "@types/pg": "^8.6.6", "@types/sinon": "^17.0.3", "@types/slug": "^5.0.8", "@types/uuid": "^9.0.1", + "@typescript-eslint/eslint-plugin": "^7.1.1", + "@typescript-eslint/parser": "^7.1.1", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "chai-http": "^4.3.0", "chai-spies": "^1.0.0", "diff-json": "^2.0.0", + "eslint": "^8.57.0", "mocha": "^10.1.0", "nock": "^13.2.9", "nodemon": "^3.0.1", diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json new file mode 100644 index 00000000..22412089 --- /dev/null +++ b/api-service/postman-collection/updated_v2_collection.json @@ -0,0 +1,2701 @@ +{ + "info": { + "_postman_id": "51471244-e9b6-453e-964d-dccbdac80e72", + "name": "V2 docs", + "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", + "_exporter_id": "37164806" + }, + "item": [ + { + "name": "Dataset api's", + "item": [ + { + "name": "Dataset create", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "response": [ + { + "name": "Success: Dataset created successfullly", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "287" + }, + { + "key": "ETag", + "value": "W/\"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ\"" + }, + { + "key": "Date", + "value": "Mon, 15 Jul 2024 13:14:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" + }, + { + "name": "Success: Master dataset created successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "291" + }, + { + "key": "ETag", + "value": "W/\"123-ZAXtVh5dFh84bZdu3SFBTQDhEHI\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:06:40 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" + }, + { + "name": "Failure: Master dataset already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "337" + }, + { + "key": "ETag", + "value": "W/\"151-a7dJ9XBUyT3AXNxl1TPcraxMX08\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:07:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" + }, + { + "name": "Failure: Dataset already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "333" + }, + { + "key": "ETag", + "value": "W/\"14d-QLXc3MJG4hFiMbBoA6mXFDpSryQ\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:08:05 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" + }, + { + "name": "Failure: Invalid request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "362" + }, + { + "key": "ETag", + "value": "W/\"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:09:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + }, + { + "name": "Success: Minimal request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "284" + }, + { + "key": "ETag", + "value": "W/\"11c-kOoTX1K7Zbp+vsGfNEv87FBJOWg\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:44:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" + }, + { + "name": "Success: Dataset created successfully with all fields", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "288" + }, + { + "key": "ETag", + "value": "W/\"120-QhzA7Fga/ADDeU90Nohadt+J8fg\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:49:53 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" + } + ] + }, + { + "name": "File generate url", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "response": [ + { + "name": "Success: Generate put url", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1344" + }, + { + "key": "ETag", + "value": "W/\"540-790rZel+H/rDwgvZRxvlUmZ8Gpc\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 02:56:19 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" + }, + { + "name": "Success: Generate get url", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1316" + }, + { + "key": "ETag", + "value": "W/\"524-iPflsPqFFV7NFaQCi2ODhQtbq/g\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 04:01:40 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" + }, + { + "name": "Failure: limit exceeds", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "355" + }, + { + "key": "ETag", + "value": "W/\"163-9oQYJJEaBH3mJAnzDHXn2MxE848\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:03:04 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid request", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "390" + }, + { + "key": "ETag", + "value": "W/\"186-KH9x0zC3+RqtWpa0tVT9XVg8agk\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 04:01:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Data schema generator", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*", + "disabled": true + }, + { + "key": "Accept-Language", + "value": "en-GB,en", + "disabled": true + }, + { + "key": "Cache-Control", + "value": "no-store", + "disabled": true + }, + { + "key": "Connection", + "value": "keep-alive", + "disabled": true + }, + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", + "disabled": true + }, + { + "key": "Origin", + "value": "http://localhost:3001", + "disabled": true + }, + { + "key": "Pragma", + "value": "no-store", + "disabled": true + }, + { + "key": "Referer", + "value": "http://localhost:3001/console/dataset/new", + "disabled": true + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty", + "disabled": true + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors", + "disabled": true + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin", + "disabled": true + }, + { + "key": "Sec-GPC", + "value": "1", + "disabled": true + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "disabled": true + }, + { + "key": "sec-ch-ua", + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", + "disabled": true + }, + { + "key": "sec-ch-ua-mobile", + "value": "?0", + "disabled": true + }, + { + "key": "sec-ch-ua-platform", + "value": "\"macOS\"", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3007/dataset/v1/dataschema" + }, + "response": [ + { + "name": "Success: Schema generated successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*", + "disabled": true + }, + { + "key": "Accept-Language", + "value": "en-GB,en", + "disabled": true + }, + { + "key": "Cache-Control", + "value": "no-store", + "disabled": true + }, + { + "key": "Connection", + "value": "keep-alive", + "disabled": true + }, + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", + "disabled": true + }, + { + "key": "Origin", + "value": "http://localhost:3001", + "disabled": true + }, + { + "key": "Pragma", + "value": "no-store", + "disabled": true + }, + { + "key": "Referer", + "value": "http://localhost:3001/console/dataset/new", + "disabled": true + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty", + "disabled": true + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors", + "disabled": true + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin", + "disabled": true + }, + { + "key": "Sec-GPC", + "value": "1", + "disabled": true + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "disabled": true + }, + { + "key": "sec-ch-ua", + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", + "disabled": true + }, + { + "key": "sec-ch-ua-mobile", + "value": "?0", + "disabled": true + }, + { + "key": "sec-ch-ua-platform", + "value": "\"macOS\"", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3007/dataset/v1/dataschema" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4414" + }, + { + "key": "ETag", + "value": "W/\"113e-ykaeY2EqBHdGqGLcr7K3WSs5fYo\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:28:22 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + } + ] + }, + { + "name": "Dataset update", + "request": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "response": [ + { + "name": "Success: Minimal dataset update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "331" + }, + { + "key": "ETag", + "value": "W/\"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 13:00:45 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" + }, + { + "name": "Success: Updated successfully", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "331" + }, + { + "key": "ETag", + "value": "W/\"14b-y8oEEJijvIDh8wU5ogipKdkv8y0\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:57:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" + }, + { + "name": "Failure: Outdated key provided", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "383" + }, + { + "key": "ETag", + "value": "W/\"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:53:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" + }, + { + "name": "Failure: Dataset not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "340" + }, + { + "key": "ETag", + "value": "W/\"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:58:30 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" + }, + { + "name": "Failure: Invalid request", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "369" + }, + { + "key": "ETag", + "value": "W/\"171-iNJoyWUecOEsXbHZwx6rld3Sr1I\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:59:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + }, + { + "name": "Failure: No fields are provided to update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "373" + }, + { + "key": "ETag", + "value": "W/\"175-ga30XLi7qSW4Ix+3Q/xaMUYcqII\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 13:02:44 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" + } + ] + }, + { + "name": "Read dataset", + "request": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "response": [ + { + "name": "Success: Read live dataset", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": "localhost:3007/v2/datasets/read/master-test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "501" + }, + { + "key": "ETag", + "value": "W/\"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" + }, + { + "name": "Success: Read draft dataset", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "528" + }, + { + "key": "ETag", + "value": "W/\"210-Qo8q3dU8l7LYIXVzJwStVe90z9Q\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:11:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" + }, + { + "name": "Success: Read specific column", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "master-test" + ], + "query": [ + { + "key": "status", + "value": "Draft" + }, + { + "key": "fields", + "value": "name,type,id" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "237" + }, + { + "key": "ETag", + "value": "W/\"ed-zvknH4AY6kid9Yit+KqMIdDeNGc\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:12:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" + }, + { + "name": "Success: Read version_key", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "fields", + "value": "version_key" + }, + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "211" + }, + { + "key": "ETag", + "value": "W/\"d3-hfyvp0pFPZhM2NbiiMDTfy+51YM\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:15:37 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" + }, + { + "name": "Success: Read from draft, if not present cerate draft from live dataset and then read", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/sample1?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "sample1" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "538" + }, + { + "key": "ETag", + "value": "W/\"21a-Iqg1rOMRqVpT0n8ZhQsPiTB2l44\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:19:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" + }, + { + "name": "Failure: Invalid field name provided", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/telemetry_record?fields=newname", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "telemetry_record" + ], + "query": [ + { + "key": "fields", + "value": "newname" + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "307" + }, + { + "key": "ETag", + "value": "W/\"133-TQ9WpmutsrDcTNkRRmbWOhUChMk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:20:17 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" + }, + { + "name": "Failure: Dataset not found", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": "localhost:3007/v2/datasets/read/new_telemetry_record.1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "304" + }, + { + "key": "ETag", + "value": "W/\"130-JL/oBB+GUHTrBp278giBHRvO71I\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:21:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" + } + ] + }, + { + "name": "Dataset list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "response": [ + { + "name": "Success: Lists all when no filters provided", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "24340" + }, + { + "key": "ETag", + "value": "W/\"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:25:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" + }, + { + "name": "Success: Filter based on status as array", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "5475" + }, + { + "key": "ETag", + "value": "W/\"1563-FnaUnwhv4LmMcE5J8f4+jKz+DDk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:27:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" + }, + { + "name": "Success: Filter basen on status as string", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "16960" + }, + { + "key": "ETag", + "value": "W/\"4240-18LMqkfsWst/M+Uc53td59PATTk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:29:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" + }, + { + "name": "Success: Filter based on dataset type", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "933" + }, + { + "key": "ETag", + "value": "W/\"3a5-IM9o2EpRUzccL+l2CW/BBpoBMqA\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:30:41 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" + }, + { + "name": "Failure: Invalid payload", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "312" + }, + { + "key": "ETag", + "value": "W/\"138-XQplwhrgIYKIg0qtQdRCYWIGTNM\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:32:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" + } + ] + }, + { + "name": "Data schema generator", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "response": [ + { + "name": "Data schema generated successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "10760" + }, + { + "key": "ETag", + "value": "W/\"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:02:50 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + }, + { + "name": "Failure: Invalid request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "364" + }, + { + "key": "ETag", + "value": "W/\"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:03:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" + }, + { + "name": "Failure: Invalid request (config not provided)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ]\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "366" + }, + { + "key": "ETag", + "value": "W/\"16e-YeX++2sGUWsjHqjP2GoTgqujU+c\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:05:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "response": [ + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "399" + }, + { + "key": "ETag", + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:43:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + }, + { + "name": "Success: Filtered based on status", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "250" + }, + { + "key": "ETag", + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:55:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + }, + { + "name": "Success: Filtered based on category", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2571" + }, + { + "key": "ETag", + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:25:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + }, + { + "name": "Success: Connectors list with all filter options", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4380" + }, + { + "key": "ETag", + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:26:32 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Connectors list without filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4380" + }, + { + "key": "ETag", + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:27:37 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "response": [ + { + "name": "Success: Read live Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "3304" + }, + { + "key": "ETag", + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 12:47:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + }, + { + "name": "Success: Read Draft Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "mssql-connector-2.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "744" + }, + { + "key": "ETag", + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 07:17:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + }, + { + "name": "Failure: Connector not found", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "276" + }, + { + "key": "ETag", + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 09:32:48 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/api-service/src/app.ts b/api-service/src/app.ts index bc7aad96..962331e9 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -1,43 +1,32 @@ import express, { Application } from "express"; -import { config } from "./v1/configs/Config"; -import { ResponseHandler } from "./v1/helpers/ResponseHandler"; -import { loadExtensions } from "./v1/managers/Extensions"; -import { router } from "./v1/routes/Router"; -import {router as v2Router} from "./v2/routes/Router" -import {router as metricsRouter} from "./v2/routes/metricRouter" +import {router as v2Router} from "./routes/Router" +import { metricRouter } from "./routes/MetricRouter" +import { druidProxyRouter } from "./routes/DruidProxyRouter" + import bodyParser from "body-parser"; -import { interceptAuditEvents } from "./v1/services/telemetry"; -import { queryService } from "./v1/routes/Router"; -import { routesConfig } from "./v1/configs/RoutesConfig"; -import { QueryValidator } from "./v1/validators/QueryValidator"; -const app: Application = express(); -const queryValidator = new QueryValidator(); +import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; +import { ResponseHandler } from "./helpers/ResponseHandler"; +import { config } from "./configs/Config"; +import { alertsRouter } from "./routes/AlertsRouter"; +import { interceptAuditEvents } from "./services/telemetry"; -const services = { - queryService, - validationService: queryValidator, - nativeQueryId: routesConfig.query.native_query.api_id, - sqlQueryId: routesConfig.query.sql_query.api_id, -} +const app: Application = express(); app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); -app.set("queryServices", services); +app.use(errorHandler) -loadExtensions(app) - .finally(() => { - app.use(interceptAuditEvents()) - app.use("/v2/", v2Router); - app.use("/", router); - app.use("/", metricsRouter); - app.use("*", ResponseHandler.routeNotFound); - app.use(ResponseHandler.errorResponse); +app.use(interceptAuditEvents()); +app.use("/v2/", v2Router); +app.use("/", druidProxyRouter); +app.use("/alerts/v1", alertsRouter); +app.use("/", metricRouter); +app.use("*", ResponseHandler.routeNotFound); +app.use(obsrvErrorHandler); - app.listen(config.api_port, () => { - console.log(`listening on port ${config.api_port}`); - }); +app.listen(config.api_port, () => { + console.log(`listening on port ${config.api_port}`); }); - -export default app; +export default app; \ No newline at end of file diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/configs/Config.ts similarity index 76% rename from api-service/src/v2/configs/Config.ts rename to api-service/src/configs/Config.ts index cdcfc8ff..b26a6906 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -15,6 +15,9 @@ export const config = { "native_query_path": "/druid/v2", "list_datasources_path": "/druid/v2/datasources", "submit_ingestion": "druid/indexer/v1/supervisor" + }, + "prometheus": { + "url": process.env.prometheus_url || "http://localhost:9090" } }, "telemetry_service_config": { @@ -49,11 +52,15 @@ export const config = { masterDataset: "master-dataset" }, "redis_config": { - "redis_host": process.env.redis_host || "localhost", - "redis_port": process.env.redis_port || 6379 + "denorm_redis_host": process.env.denorm_redis_host || "localhost", + "denorm_redis_port": parseInt(process.env.denorm_redis_port as string) || 6379, + "dedup_redis_host": process.env.dedup_redis_host || "localhost", + "dedup_redis_port": parseInt(process.env.dedup_redis_port as string) || 6379 }, "exclude_datasource_validation": process.env.exclude_datasource_validation ? process.env.exclude_datasource_validation.split(",") : ["system-stats", "failed-events-summary", "masterdata-system-stats", "system-events"], // list of datasource names to skip validation while calling query API "telemetry_dataset": process.env.telemetry_dataset || `${env}.system.telemetry.events`, + "rollup_ratio": parseInt(process.env.rollup_ratio || "80"), + "unique_formats": ["uuid", "email", "uri", "ipv4", "ipv6"], "table_config": { // This object defines the configuration for each table. "datasets": { "primary_key": "id", @@ -87,5 +94,25 @@ export const config = { "read_storage_url_expiry": process.env.read_storage_url_expiry ? parseInt(process.env.read_storage_url_expiry) : 600, "write_storage_url_expiry": process.env.write_storage_url_expiry ? parseInt(process.env.write_storage_url_expiry) : 600, "service": process.env.service || "api-service" - } + }, + "command_service_config": { + "host": process.env.command_service_host || "http://localhost", + "port": parseInt(process.env.command_service_port || "8000"), + "path": process.env.command_service_path || "/system/v1/dataset/command" + }, + "flink_job_configs": { + "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", + "masterdata_processor_job_manager_url": process.env.masterdata_processor_job_manager_url || "http://localhost:8081" + }, + "encryption_config": { + "encryption_key": process.env.encryption_key || "strong_encryption_key_to_encrypt", + "encryption_algorithm": process.env.encryption_algorithm || "aes-256-ecb", + }, + "grafana_config": { + "dialect": process.env.dialet || "postgres", + "url": process.env.grafana_url || "http://localhost:8000", + "access_token": process.env.grafana_token || "" + }, + "user_token_public_key": process.env.user_token_public_key || "", + "is_RBAC_enabled": process.env.is_rbac_enabled || "false", } diff --git a/api-service/src/v2/configs/ConnectionsConfig.ts b/api-service/src/configs/ConnectionsConfig.ts similarity index 100% rename from api-service/src/v2/configs/ConnectionsConfig.ts rename to api-service/src/configs/ConnectionsConfig.ts diff --git a/api-service/src/configs/DatasetConfigDefault.ts b/api-service/src/configs/DatasetConfigDefault.ts new file mode 100644 index 00000000..27bdeda0 --- /dev/null +++ b/api-service/src/configs/DatasetConfigDefault.ts @@ -0,0 +1,60 @@ +import { config } from "./Config"; +import { DatasetStatus, ValidationMode } from "../types/DatasetModels"; +import { ingestionConfig } from "./IngestionConfig"; + +export const defaultDatasetConfig = { + "validation_config": { + "validate": true, + "mode": ValidationMode.Strict, + }, + "extraction_config": { + "is_batch_event": true, + "extraction_key": "events", + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "id", + "dedup_period": 604800, // 7 days + } + }, + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "id", + "dedup_period": 604800, // 7 days + }, + "denorm_config": { + "redis_db_host": config.redis_config.denorm_redis_host, + "redis_db_port": config.redis_config.denorm_redis_port, + "denorm_fields": [] + }, + "router_config": { + "topic": "" + }, + "tags": [], + "dataset_config": { + "indexing_config": { + "olap_store_enabled": true, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "data_key": "", + "partition_key": "", + "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"] + }, + "cache_config": { + "redis_db_host": config.redis_config.denorm_redis_host, + "redis_db_port": config.redis_config.denorm_redis_port, + "redis_db": 0 + }, + "file_upload_path": [] + }, + "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, + "status": DatasetStatus.Draft, + "api_version": "v2", + "sample_data":{}, + "version": 1, + "created_by": "SYSTEM", + "updated_by": "SYSTEM" +} + +export const validDatasetFields = ["dataset_id", "id", "name", "type", "validation_config", "extraction_config", "dedup_config", "data_schema", "router_config", "denorm_config", "transformations_config", "dataset_config", "tags", "status", "version", "created_by", "updated_by", "created_date", "updated_date", "published_date", "version_key"] \ No newline at end of file diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts new file mode 100644 index 00000000..8826b264 --- /dev/null +++ b/api-service/src/configs/IngestionConfig.ts @@ -0,0 +1,79 @@ +import { config } from "./Config"; + +const env = process.env; + +export const ingestionConfig = { + "indexCol": { "Event Arrival Time": "obsrv_meta.syncts" }, + "granularitySpec": { + "rollup": false, + "segmentGranularity": env.segment_granularity || "DAY" + }, + "ioconfig": { "topic": "", "taskDuration": "PT1H" }, + "tuningConfig": { "maxRowPerSegment": 5000000, "taskCount": 1 }, + "query_granularity": "none", + "use_earliest_offset": true, + "completion_timeout": "PT1H", + "maxBytesInMemory": env.max_bytes_in_memory || 134217728, + "syncts_path": "$.obsrv_meta.syncts", +} + +export const rawIngestionSpecDefaults = { + "granularitySpec": { + "type": "uniform", + "rollup": false, + "segmentGranularity": env.segment_granularity || "DAY", + "queryGranularity": "none" + }, + "tuningConfig": { + "type": "kafka", + "maxBytesInMemory": env.max_bytes_in_memory || 134217728, + "maxRowsPerSegment": env.max_rows_per_segment || 5000000, + "logParseExceptions": true + }, + "ioConfig": { + "type": "kafka", + "topic": "", + "consumerProperties": { "bootstrap.servers": config.telemetry_service_config.kafka.config.brokers[0] }, + "taskCount": env.supervisor_task_count || 1, + "replicas": 1, + "taskDuration": env.default_task_duration || "PT4H", + "useEarliestOffset": true, + "completionTimeout": env.default_task_duration || "PT4H", + "inputFormat": { + "type": "json", + "flattenSpec": { + "useFieldDiscovery": true + } + }, + "appendToExisting": false + }, + "synctsField": { + "name": "obsrv_meta.syncts", + "arrival_format": "text", + "data_type": "date", + "type": "text", + "expr": "$.obsrv_meta.syncts" + }, + "dimensions": [ + { + "type": "string", + "name": "obsrv.meta.source.connector" + }, + { + "type": "string", + "name": "obsrv.meta.source.id" + } + ], + "flattenSpec": [ + { + "type": "path", + "expr": "$.obsrv_meta.source.connector", + "name": "obsrv.meta.source.connector" + }, + { + "type": "path", + "expr": "$.obsrv_meta.source.connectorInstance", + "name": "obsrv.meta.source.id" + } + ] +} \ No newline at end of file diff --git a/api-service/src/v1/configs/QueryRules.ts b/api-service/src/configs/QueryRules.ts similarity index 100% rename from api-service/src/v1/configs/QueryRules.ts rename to api-service/src/configs/QueryRules.ts diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts new file mode 100644 index 00000000..94597474 --- /dev/null +++ b/api-service/src/connections/commandServiceConnection.ts @@ -0,0 +1,21 @@ +import axios from "axios"; +import _ from "lodash"; +import { config } from "../configs/Config"; +import { v4 } from "uuid"; + +const commandHost = _.get(config, ["command_service_config", "host"]) +const commandPort = _.get(config, ["command_service_config", "port"]) +const commandPath = _.get(config, ["command_service_config", "path"]) + +export const commandHttpService = axios.create({ baseURL: `${commandHost}:${commandPort}`, headers: { "Content-Type": "application/json" } }); + +export const executeCommand = async (id: string, command: string) => { + const payload = { + "id": v4(), + "data": { + "dataset_id": id, + "command": command + } + } + return commandHttpService.post(commandPath, payload) +} \ No newline at end of file diff --git a/api-service/src/v2/connections/databaseConnection.ts b/api-service/src/connections/databaseConnection.ts similarity index 70% rename from api-service/src/v2/connections/databaseConnection.ts rename to api-service/src/connections/databaseConnection.ts index c9f4c89c..4adc7be0 100644 --- a/api-service/src/v2/connections/databaseConnection.ts +++ b/api-service/src/connections/databaseConnection.ts @@ -4,9 +4,18 @@ import { connectionConfig } from "../configs/ConnectionsConfig" const { database, host, password, port, username } = connectionConfig.postgres export const sequelize = new Sequelize({ - database, password, username: username, dialect: "postgres", host, port: +port + database, password, username: username, dialect: "postgres", host, port: +port, pool: { + max: 2, + min: 1, + acquire: 30000, + idle: 10000 + } }) + export const health = async () => { + return sequelize.query("select 1") +} + export const query = async (query: string) => { const [results, metadata] = await sequelize.query(query) return { diff --git a/api-service/src/v2/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts similarity index 84% rename from api-service/src/v2/connections/druidConnection.ts rename to api-service/src/connections/druidConnection.ts index 8ae95679..7ca7b2dc 100644 --- a/api-service/src/v2/connections/druidConnection.ts +++ b/api-service/src/connections/druidConnection.ts @@ -19,4 +19,6 @@ export const executeSqlQuery = async (payload: any) => { export const getDatasourceListFromDruid = async () => { const existingDatasources = await axios.get(`${config?.query_api?.druid?.host}:${config?.query_api?.druid?.port}${config.query_api.druid.list_datasources_path}`, {}) return existingDatasources; -} \ No newline at end of file +} + +export const druidHttpService = axios.create({ baseURL: `${config.query_api.druid.host}:${config.query_api.druid.port}`, headers: { "Content-Type": "application/json" } }); \ No newline at end of file diff --git a/api-service/src/connections/grafanaConnection.ts b/api-service/src/connections/grafanaConnection.ts new file mode 100644 index 00000000..b70e6009 --- /dev/null +++ b/api-service/src/connections/grafanaConnection.ts @@ -0,0 +1,10 @@ +import axios from "axios"; +import { config } from "../configs/Config"; + +const grafanaHttpClient = axios.create({ + baseURL: config.grafana_config.url +}); + +grafanaHttpClient.defaults.headers.common["Authorization"] = config.grafana_config.access_token; + +export { grafanaHttpClient }; \ No newline at end of file diff --git a/api-service/src/v2/connections/kafkaConnection.ts b/api-service/src/connections/kafkaConnection.ts similarity index 79% rename from api-service/src/v2/connections/kafkaConnection.ts rename to api-service/src/connections/kafkaConnection.ts index 24b396a5..ff9ce0ca 100644 --- a/api-service/src/v2/connections/kafkaConnection.ts +++ b/api-service/src/connections/kafkaConnection.ts @@ -39,4 +39,18 @@ const send = async (payload: Record, topic: string) => { } } -export { connect, send } +const admin = kafka.admin(); + +const isHealthy = async () => { + try { + await admin.connect(); + return true + } catch (error) { + logger.debug("Failed to connect to Kafka:", error); + } finally { + await admin.disconnect(); + } + return false; +} + +export { connect, send, isHealthy } diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts new file mode 100644 index 00000000..278ac586 --- /dev/null +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -0,0 +1,145 @@ +import { Request, Response, NextFunction } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { Alert } from "../../models/Alert"; +import httpStatus from "http-status"; +import errorResponse from "http-errors"; +import { deleteAlertRule, getAlertPayload, getAlertRule, getAlertsMetadata, publishAlert, retireAlertSilence, deleteSystemRules } from "../../services/managers"; +import _ from "lodash"; + +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "alert", ver: "1.0.0" }; + +const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const alertPayload = getAlertPayload(req.body); + const response = await Alert.create(alertPayload); + updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); + } catch (error: any) { + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const publishAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const rulePayload: Record | null = await getAlertRule(alertId); + if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + if (rulePayload.status == "live") { + await deleteAlertRule(rulePayload, false); + } + await publishAlert(rulePayload); + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); + } catch (error: any) { + console.log(error?.message) + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +}; + +const transformAlerts = async (alertModel: any) => { + const alert = alertModel?.toJSON(); + const status = _.get(alert, "status"); + if (status !== "live") return alert; + return getAlertsMetadata(alert); +} + +const searchAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { limit, filters, offset, options = {} } = req.body?.request || {}; + const alerts = await Alert.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }), ...options }); + const alertRulesWithStatus = await Promise.all(_.map(alerts, transformAlerts)); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { alerts: alertRulesWithStatus, count: alerts.length } }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +}; + +const alertDetailsHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const ruleModel: Record | null = await getAlertRule(alertId); + if (!ruleModel) { + return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + } + let rulePayload = ruleModel.toJSON(); + if (rulePayload.status == "live") { + rulePayload = await getAlertsMetadata(rulePayload) + } + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { alerts: rulePayload } }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +} + +const deleteAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const { hardDelete = "false" } = req.query + const ruleModel = await getAlertRule(alertId); + if (!ruleModel) { + return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + } + const rulePayload = ruleModel.toJSON(); + await deleteAlertRule(rulePayload, hardDelete === "true"); + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +} + +const updateAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const isEmpty = _.isEmpty(req.body); + if (isEmpty) throw new Error("Failed to update record"); + const ruleModel = await getAlertRule(alertId); + if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } + const rulePayload = ruleModel.toJSON(); + if (rulePayload.status == "live") { + await deleteAlertRule(rulePayload, false); + await retireAlertSilence(alertId); + } + const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); + await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } }); + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); + } catch (error: any) { + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const deleteSystemAlertsHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const body = req.body; + const { filters } = body; + if (!filters) throw new Error("Failed to update record"); + await deleteSystemRules({ filters, manager: "grafana" }); + await Alert.destroy({ where: filters }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +} + +export default { + alertDetailsHandler, + searchAlertHandler, + publishAlertHandler, + createAlertHandler, + deleteAlertHandler, + updateAlertHandler, + deleteSystemAlertsHandler +} + diff --git a/api-service/src/controllers/Alerts/Metric.ts b/api-service/src/controllers/Alerts/Metric.ts new file mode 100644 index 00000000..6f80f16f --- /dev/null +++ b/api-service/src/controllers/Alerts/Metric.ts @@ -0,0 +1,86 @@ +import _ from "lodash"; +import { Request, Response, NextFunction } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { Metrics } from "../../models/Metric"; +import httpStatus from "http-status"; +import errorResponse from "http-errors"; +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "metric", ver: "1.0.0" }; + +const createMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { component } = req.body; + const transformComponent = _.toLower(component); + const metricsBody = await Metrics.create({ ...(req.body), component: transformComponent }); + updateTelemetryAuditEvent({ request: req, object: { id: metricsBody?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: metricsBody.dataValues.id } }); + } catch (error: any) { + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const listMetricsHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { limit, filters, offset } = _.get(req.body, "request") || {}; + const metricsPayload = await Metrics.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { metrics: metricsPayload, count: metricsPayload.length } }); + } catch (error) { + const errorMessage = _.get(error, "message") + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const updateMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const toUpdatePayload = req.body; + const { component } = toUpdatePayload; + const isEmpty = _.isEmpty(toUpdatePayload); + if (isEmpty) throw new Error("Failed to update record"); + const record = await Metrics.findOne({ where: { id } }); + if (!record) throw new Error(httpStatus[httpStatus.NOT_FOUND]); + updateTelemetryAuditEvent({ request: req, object: { id, ...telemetryObject }, currentRecord: record }) + await Metrics.update({ ...toUpdatePayload, ...(component) && { component: _.toLower(component) } }, { + where: { id } + }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); + } catch (error) { + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const deleteMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const record = await Metrics.findOne({ where: { id } }); + if (!record) throw new Error(httpStatus[httpStatus.NOT_FOUND]); + await record.destroy(); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); + } catch (error) { + const errorMessage = _.get(error, "message") + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const deleteMultipleMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { filters } = req.body; + if (!filters) throw new Error("Failed to update record"); + await Metrics.destroy({ where: filters }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} }); + } catch (error) { + const errorMessage = _.get(error, "message") + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +export default { createMetricHandler, listMetricsHandler, updateMetricHandler, deleteMetricHandler, deleteMultipleMetricHandler }; \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts new file mode 100644 index 00000000..32e2c531 --- /dev/null +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -0,0 +1,117 @@ +import { Request, Response, NextFunction } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { createSilence, deleteSilence, getSilenceMetaData, updateSilence } from "../../services/managers"; +import httpStatus from "http-status"; +import errorResponse from "http-errors"; +import _ from "lodash"; +import { Silence } from "../../models/Silence"; +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "alert-silence", ver: "1.0.0" }; + +const createHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const payload = request.body; + const { startDate, endDate, alertId } = payload; + const existingSilence = await Silence.findOne({ where: { alert_id: alertId } }); + if (existingSilence) existingSilence.destroy(); + const grafanaResponse = await createSilence(payload); + if (!grafanaResponse) return next({ message: httpStatus[httpStatus.INTERNAL_SERVER_ERROR], statusCode: httpStatus.INTERNAL_SERVER_ERROR }) + + const start_date = new Date(startDate); + const end_date = new Date(endDate); + const silenceBody = { + id: grafanaResponse.silenceId, + manager: grafanaResponse.manager, + alert_id: alertId, + start_time: start_date, + end_time: end_date, + } + const sileneResponse = await Silence.create(silenceBody); + updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: sileneResponse.dataValues.id } }) + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const transformSilences = async (silenceModel: any) => { + const silence = silenceModel?.toJSON(); + return getSilenceMetaData(silence); +} + + +const listHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const silences = await Silence.findAll(); + const count = _.get(silences, "length"); + const transformedSilences = await Promise.all(silences.map(transformSilences)); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { transformedSilences, ...(count && { count }) } }); + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const fetchHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const id = request.params.id; + const silenceModel = await Silence.findOne({ where: { id } }); + const transformedSilence = await transformSilences(silenceModel); + if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: transformedSilence }); + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const updateHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const id = request.params.id; + const payload = request.body; + const silenceModel = await Silence.findOne({ where: { id } }); + if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const silenceObject = silenceModel?.toJSON(); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject }); + await updateSilence(silenceObject, payload); + const updatedStartTime = new Date(payload.startTime); + const updatedEndTime = new Date(payload.endTime); + const updatedSilence = { + ...silenceObject, + start_time: updatedStartTime, + end_time: updatedEndTime + } + const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const deleteHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const id = request.params.id; + const silenceModel = await Silence.findOne({ where: { id } }); + if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const silenceObject = silenceModel?.toJSON(); + if (silenceObject?.status === "expired") return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST }); + await deleteSilence(silenceObject); + await silenceModel.destroy(); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }) + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +export default { + createHandler, + listHandler, + fetchHandler, + updateHandler, + deleteHandler +} \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts new file mode 100644 index 00000000..1dadd5ab --- /dev/null +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import ConnectorListSchema from "./ConnectorsListValidationSchema.json"; +import { obsrvError } from "../../types/ObsrvError"; +import { schemaValidation } from "../../services/ValidationService"; +import _ from "lodash"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import { connectorService } from "../../services/ConnectorService"; + +const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date", "live_date"]; + +const validateRequest = (req: Request) => { + const isRequestValid: Record = schemaValidation(req.body, ConnectorListSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "CONNECTORS_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) + } +} + +const connectorsList = async (req: Request, res: Response) => { + validateRequest(req); + const connectorBody = _.get(req, ["body", "request"]); + const connectorList = await listConnectors(connectorBody) + const responseData = { data: connectorList, count: _.size(connectorList) } + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); +} + +const listConnectors = async (request: Record): Promise> => { + const { filters = {} } = request || {}; + const status = _.get(filters, "status"); + const category = _.get(filters, "category"); + const filterOptions: any = {}; + if (!_.isEmpty(status)) filterOptions["status"] = status; + if (!_.isEmpty(category)) filterOptions["category"] = category; + return connectorService.findConnectors(filterOptions, defaultFields); + +} + +export default connectorsList; \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json b/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json new file mode 100644 index 00000000..61847b06 --- /dev/null +++ b/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json @@ -0,0 +1,73 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "api.connectors.list" + ] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "filters": { + "type": "object", + "properties": { + "status": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "Draft", + "InValidation", + "Live", + "Retired" + ] + }, + "minItems": 1 + }, + "category": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "Database", + "File" + ] + }, + "minItems": 1 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": [ + "id", + "ver", + "ts", + "params", + "request" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts new file mode 100644 index 00000000..1e482e2c --- /dev/null +++ b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts @@ -0,0 +1,22 @@ +import { Request, Response } from "express"; +import { obsrvError } from "../../types/ObsrvError"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import { connectorService } from "../../services/ConnectorService"; +import _ from "lodash"; + +const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "licence", "owner", "iconurl", "status", "ui_spec", "created_by", "updated_by", "created_date", "updated_date", "live_date"]; + +const connectorsRead = async (req: Request, res: Response) => { + const { id } = req.params; + const { mode } = req.query; + const isEditMode = _.toLower(_.toString(mode)) === "edit" + const status = isEditMode ? "Draft" : "Live" + const connector = await connectorService.getConnector({ id, status }, defaultFields) + if (!connector) { + throw obsrvError("", "CONNECTOR_NOT_FOUND", `Connector not found: ${id}`, "NOT_FOUND", 404); + } + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: connector }); +} + +export default connectorsRead; \ No newline at end of file diff --git a/api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateController.ts rename to api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts diff --git a/api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json rename to api-service/src/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json diff --git a/api-service/src/v2/controllers/CreateQueryTemplate/QueryTemplateValidator.ts b/api-service/src/controllers/CreateQueryTemplate/QueryTemplateValidator.ts similarity index 100% rename from api-service/src/v2/controllers/CreateQueryTemplate/QueryTemplateValidator.ts rename to api-service/src/controllers/CreateQueryTemplate/QueryTemplateValidator.ts diff --git a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/controllers/DataExhaust/DataExhaustController.ts similarity index 94% rename from api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts rename to api-service/src/controllers/DataExhaust/DataExhaustController.ts index d8cc88d1..ceeb3358 100644 --- a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts +++ b/api-service/src/controllers/DataExhaust/DataExhaustController.ts @@ -4,7 +4,7 @@ import httpStatus from "http-status"; import { getDateRange, isValidDateRange } from "../../utils/common"; import { config } from "../../configs/Config"; import moment from "moment"; -import { getDataset, setReqDatasetId } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; import * as _ from "lodash"; import logger from "../../logger"; import { cloudProvider } from "../../services/CloudServices"; @@ -14,10 +14,9 @@ export const dataExhaust = async (req: Request, res: Response) => { const { datasetId } = params; const { type }: any = req.query; const momentFormat = "YYYY-MM-DD"; - setReqDatasetId(req, datasetId) const verifyDatasetExists = async (datasetId: string) => { - const dataset = await getDataset(datasetId) + const dataset = await datasetService.getDataset(datasetId, ["id"], true) return dataset; } diff --git a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts similarity index 67% rename from api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts rename to api-service/src/controllers/DataIngestion/DataIngestionController.ts index 15f80da4..46f829e9 100644 --- a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -4,7 +4,7 @@ import validationSchema from "./validationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { send } from "../../connections/kafkaConnection"; -import { getDataset, setReqDatasetId } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; import logger from "../../logger"; import { config } from "../../configs/Config"; @@ -23,42 +23,31 @@ const errorObject = { } } const apiId = "api.data.in"; -const errorCode = "DATASET_UPDATE_FAILURE" const dataIn = async (req: Request, res: Response) => { - try { - const requestBody = req.body; + + const requestBody = req.body; const datasetId = req.params.datasetId.trim(); - setReqDatasetId(req, datasetId) - + const isValidSchema = schemaValidation(requestBody, validationSchema) if (!isValidSchema?.isValid) { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); } - const dataset = await getDataset(datasetId) + const dataset = await datasetService.getDataset(datasetId, ["id", "entry_topic", "api_version", "dataset_config"], true) if (!dataset) { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); } - const entryTopic = _.get(dataset, "dataValues.dataset_config.entry_topic") + const { entry_topic, dataset_config, api_version } = dataset + const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic if (!entryTopic) { logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); } - await send(addMetadataToEvents(datasetId, requestBody), _.get(dataset, "dataValues.dataset_config.entry_topic")) + await send(addMetadataToEvents(datasetId, requestBody), entryTopic) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); - } - catch (err: any) { - const code = _.get(err, "code") || errorCode - logger.error({ ...err, apiId, code }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "DATA_INGESTION_FAILED", message: "Failed to ingest data" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } + } const addMetadataToEvents = (datasetId: string, payload: any) => { @@ -69,18 +58,23 @@ const addMetadataToEvents = (datasetId: string, payload: any) => { const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source }; if (Array.isArray(validData)) { const payloadRef = validData.map((event: any) => { - event = _.set(event, "obsrv_meta", obsrvMeta); - event = _.set(event, "dataset", datasetId); - event = _.set(event, "msgid", mid); - return event + const payload = { + event, + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + } + return payload; }) return payloadRef; } else { - _.set(validData, "msgid", mid); - _.set(validData, "obsrv_meta", obsrvMeta); - _.set(validData, "dataset", datasetId); - return validData + return ({ + "event": validData, + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + }); } } diff --git a/api-service/src/v2/controllers/DataIngestion/validationSchema.json b/api-service/src/controllers/DataIngestion/validationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DataIngestion/validationSchema.json rename to api-service/src/controllers/DataIngestion/validationSchema.json diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts new file mode 100644 index 00000000..dd6f93b6 --- /dev/null +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -0,0 +1,45 @@ +import { Request, Response } from "express"; +import logger from "../../logger"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { schemaValidation } from "../../services/ValidationService"; +import validationSchema from "./DataOutValidationSchema.json"; +import { validateQuery } from "./QueryValidator"; +import * as _ from "lodash"; +import { executeNativeQuery, executeSqlQuery } from "../../connections/druidConnection"; + +export const apiId = "api.data.out"; +const dataOut = async (req: Request, res: Response) => { + const datasetId = req.params?.datasetId; + const requestBody = req.body; + const msgid = _.get(req, "body.params.msgid"); + const isValidSchema = schemaValidation(requestBody, validationSchema); + if (!isValidSchema?.isValid) { + logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); + } + const isValidQuery: any = await validateQuery(req.body, datasetId); + const query = _.get(req, "body.query", "") + + if (isValidQuery === true && _.isObject(query)) { + const result = await executeNativeQuery(query); + logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); + } + + if (isValidQuery === true && _.isString(query)) { + const result = await executeSqlQuery({ query }) + logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); + } + + else { + logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) + return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); + } +} + +export default dataOut; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DataOut/DataOutValidationSchema.json b/api-service/src/controllers/DataOut/DataOutValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DataOut/DataOutValidationSchema.json rename to api-service/src/controllers/DataOut/DataOutValidationSchema.json diff --git a/api-service/src/v2/controllers/DataOut/QueryRules.ts b/api-service/src/controllers/DataOut/QueryRules.ts similarity index 100% rename from api-service/src/v2/controllers/DataOut/QueryRules.ts rename to api-service/src/controllers/DataOut/QueryRules.ts diff --git a/api-service/src/v2/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts similarity index 100% rename from api-service/src/v2/controllers/DataOut/QueryValidator.ts rename to api-service/src/controllers/DataOut/QueryValidator.ts diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts new file mode 100644 index 00000000..9308dd71 --- /dev/null +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -0,0 +1,53 @@ + +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import * as _ from "lodash"; +import { schemaValidation } from "../../services/ValidationService"; +import validationSchema from "./RequestValidationSchema.json"; +import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import { updateRecords } from "./DatasetCopyHelper"; +import { obsrvError } from "../../types/ObsrvError"; + +export const apiId = "api.dataset.copy"; + +const validateRequest = (req: Request) => { + + const isValidSchema = schemaValidation(req.body, validationSchema); + if (!isValidSchema?.isValid) { + throw obsrvError("", "DATASET_COPY_INVALID_INPUT", isValidSchema.message, "BAD_REQUEST", 400); + } +} + +const fetchDataset = async (req: Request) => { + const datasetId = _.get(req, "body.request.source.datasetId"); + const isLive = _.get(req, "body.request.source.isLive"); + + let dataset = isLive ? await getLiveDatasetConfigs(datasetId) : await datasetService.getDraftDataset(datasetId) + if (!dataset) { + throw obsrvError(datasetId, "DATASET_NOT_EXISTS", `Dataset ${datasetId} does not exists`, "NOT_FOUND", 404); + } + + if (_.get(dataset, "api_version") != "v2") { + const migratedConfigs = await datasetService.migrateDatasetV1(datasetId, dataset) + dataset = { ...dataset, ...migratedConfigs } + } + + return dataset; +} + +const datasetCopy = async (req: Request, res: Response) => { + + validateRequest(req); + const newDatasetId = _.get(req, "body.request.destination.datasetId"); + const dataset = await fetchDataset(req); + updateRecords(dataset, newDatasetId) + const response = await datasetService.createDraftDataset(dataset).catch(err => { + if (err?.name === "SequelizeUniqueConstraintError") { + throw obsrvError(newDatasetId, "DATASET_ALREADY_EXISTS", `Dataset with id ${newDatasetId} already exists`, "BAD_REQUEST", 400); + } + throw obsrvError(newDatasetId, "DATASET_COPY_FAILURE", `Failed to clone dataset`, "INTERNAL_SERVER_ERROR", 500); + }); + return ResponseHandler.successResponse(req, res, { status: 200, data: { dataset_id: _.get(response, "id"), message: `Dataset clone successful` } }); +} + +export default datasetCopy; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts new file mode 100644 index 00000000..3ce8b0fe --- /dev/null +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -0,0 +1,17 @@ +import * as _ from "lodash"; +import { DatasetStatus } from "../../types/DatasetModels"; +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { config } from "../../configs/Config"; +const version = defaultDatasetConfig.version; + +export const updateRecords = (datasetRecord: Record, newDatasetId: string): void => { + const dataset_id = newDatasetId; + _.set(datasetRecord, "api_version", "v2") + _.set(datasetRecord, "status", DatasetStatus.Draft) + _.set(datasetRecord, "dataset_id", dataset_id) + _.set(datasetRecord, "id", dataset_id) + _.set(datasetRecord, "version_key", Date.now().toString()) + _.set(datasetRecord, "version", version); + _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) + _.set(datasetRecord, "router_config", { topic: newDatasetId }) +} diff --git a/api-service/src/controllers/DatasetCopy/RequestValidationSchema.json b/api-service/src/controllers/DatasetCopy/RequestValidationSchema.json new file mode 100644 index 00000000..3109df60 --- /dev/null +++ b/api-service/src/controllers/DatasetCopy/RequestValidationSchema.json @@ -0,0 +1,57 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.copy"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "source": { + "type": "object", + "properties": { + "datasetId": { + "type": "string", + "minLength": 3 + }, + "isLive": { + "type": "boolean" + } + }, + "required": ["datasetId", "isLive"] + }, + "destination": { + "type": "object", + "properties": { + "datasetId": { + "type": "string", + "minLength": 3 + } + }, + "required": ["datasetId"] + } + }, + "required": ["source", "destination"], + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts new file mode 100644 index 00000000..2867457b --- /dev/null +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -0,0 +1,91 @@ +import _ from "lodash"; +import { Request, Response } from "express"; +import httpStatus from "http-status"; +import { datasetService } from "../../services/DatasetService"; +import DatasetCreate from "./DatasetCreateValidationSchema.json"; +import { schemaValidation } from "../../services/ValidationService"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { cipherService } from "../../services/CipherService"; +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { obsrvError } from "../../types/ObsrvError"; + +export const apiId = "api.datasets.create" + +const validateRequest = async (req: Request) => { + + const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } + const datasetId = _.get(req, ["body", "request", "dataset_id"]) + const isDataSetExists = await datasetService.checkDatasetExists(datasetId); + if (isDataSetExists) { + throw obsrvError(datasetId, "DATASET_EXISTS", `Dataset Already exists with id:${datasetId}`, "CONFLICT", 409) + } + + const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"])) + if (!_.isEmpty(duplicateDenormKeys)) { + throw obsrvError(datasetId, "DATASET_DUPLICATE_DENORM_KEY", "Duplicate denorm output fields found.", "BAD_REQUEST", 400, undefined, {duplicateKeys: duplicateDenormKeys}) + } + +} + +const datasetCreate = async (req: Request, res: Response) => { + + await validateRequest(req) + const draftDataset = getDraftDataset(req.body.request) + const dataset = await datasetService.createDraftDataset(draftDataset); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); +} + +const getDraftDataset = (datasetReq: Record): Record => { + const transformationsConfig:Array> = _.get(datasetReq, "transformations_config") + const connectorsConfig:Array> = _.get(datasetReq, "connectors_config") + const dataset = _.omit(datasetReq, ["transformations_config", "connectors_config"]) + const mergedDataset = mergeDatasetConfigs(defaultDatasetConfig, dataset) + const draftDataset = { + ...mergedDataset, + version_key: Date.now().toString(), + transformations_config: getDatasetTransformations(transformationsConfig), + connectors_config: getDatasetConnectors(connectorsConfig), + } + return draftDataset; +} + +const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { + const { id, dataset_id } = requestPayload; + const datasetId = !id ? dataset_id : id + const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } + const defaults = _.cloneDeep(defaultConfig) + const datasetConfigs = _.merge(defaults, modifyPayload) + return datasetConfigs +} + +const getDatasetConnectors = (connectorConfigs: Array>): Array> => { + + if (!_.isEmpty(connectorConfigs)) { + const uniqueConnectors = _.uniqWith(connectorConfigs, (a: Record, b: Record) => { + return _.isEqual(a.connector_id, b.connector_id) && _.isEqual(a.connector_config, b.connector_config) + }) + return _.map(uniqueConnectors, (config) => { + return { + id: config.id, + connector_id: config.connector_id, + connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), + operations_config: config.operations_config, + version: config.version + } + }) + } + return [] +} + +const getDatasetTransformations = (configs: Array>): Array> => { + + if (configs) { + return _.uniqBy(configs, "field_key") + } + return [] +} + +export default datasetCreate; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json similarity index 52% rename from api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json rename to api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json index d1a27ed7..13097a8b 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -3,7 +3,7 @@ "properties": { "id": { "type": "string", - "enum": ["api.datasets.update"] + "enum": ["api.datasets.create"] }, "ver": { "type": "string" @@ -23,14 +23,14 @@ }, "request": { "type": "object", - "additionalProperties": false, "properties": { "dataset_id": { "type": "string", "minLength": 1 }, - "version_key": { - "type": "string" + "type": { + "type": "string", + "enum": ["event", "transaction", "master"] }, "name": { "type": "string", @@ -40,15 +40,17 @@ "type": "object", "properties": { "validate": { - "type": "boolean" + "type": "boolean", + "default": true }, "mode": { "type": "string", - "enum": ["Strict", "IgnoreNewFields"] + "enum": ["Strict", "IgnoreNewFields"], + "default": "Strict" } }, - "additionalProperties": false, "required": ["validate"], + "additionalProperties": false, "if": { "properties": { "validate": { @@ -64,21 +66,22 @@ "type": "object", "properties": { "is_batch_event": { - "type": "boolean" + "type": "boolean", + "default": true }, "extraction_key": { "type": "string", - "minLength": 1 + "default": "events" }, "dedup_config": { "type": "object", "properties": { "drop_duplicates": { - "type": "boolean" + "type": "boolean", + "default": false }, "dedup_key": { - "type": "string", - "minLength": 1 + "type": "string" } }, "if": { @@ -89,8 +92,14 @@ } }, "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, "required": ["dedup_key"] }, + "required": ["drop_duplicates"], "additionalProperties": false } }, @@ -104,6 +113,11 @@ } }, "then": { + "properties": { + "extraction_key": { + "minLength": 1 + } + }, "required": ["extraction_key", "dedup_config"] } }, @@ -111,14 +125,13 @@ "type": "object", "properties": { "drop_duplicates": { - "type": "boolean" + "type": "boolean", + "default": true }, "dedup_key": { - "type": "string", - "minLength": 1 + "type": "string" } }, - "additionalProperties": false, "if": { "properties": { "drop_duplicates": { @@ -127,8 +140,15 @@ } }, "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, "required": ["dedup_key"] - } + }, + "required": ["drop_duplicates"], + "additionalProperties": false }, "data_schema": { "type": "object", @@ -156,92 +176,113 @@ "dataset_config": { "type": "object", "properties": { - "data_key": { - "type": "string", - "minLength": 1 - }, - "timestamp_key": { - "type": "string", - "minLength": 1 - }, "file_upload_path": { "type": "array", "items": { "type": "string", "minLength": 1 } + }, + "dataset_tz": { + "type": "string" + }, + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false } }, + "required": ["indexing_config", "keys_config"], "additionalProperties": false }, - "transformation_config": { + "transformations_config": { "type": "array", "items": { "type": "object", "properties": { - "values": { + "field_key": { + "type": "string", + "minLength": 1 + }, + "transformation_function": { "type": "object", "properties": { - "field_key": { + "type": { "type": "string", "minLength": 1 }, - "transformation_function": { - "type": "object" - }, - "mode": { + "expr": { "type": "string", - "enum": ["Strict", "Lenient"] + "minLength": 1 + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false }, - "metadata": { - "type": "object" + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] } }, - "additionalProperties": false, - "required": ["field_key"] + "required": ["type", "expr"], + "additionalProperties": false }, - "action": { + "mode": { "type": "string", - "enum": ["add", "remove", "update"] + "enum": [ + "Strict", + "Lenient" + ] } }, "additionalProperties": false, - "required": ["values", "action"], - "if": { - "properties": { - "action": { - "const": "add" - } - } - }, - "then": { - "properties": { - "values": { - "required": ["transformation_function", "mode"] - } - } - } - } - }, - "tags": { - "type": "array", - "items": { - "type": "object", - "properties": { - "values": { - "type": "array", - "items": { - "type": "string", - "minLength":1 - }, - "minItems": 1 - }, - "action": { - "type": "string", - "enum": ["add", "remove"] - } - }, - "required": ["values", "action"] + "required": ["field_key", "transformation_function", "mode"] } }, "denorm_config": { @@ -252,47 +293,80 @@ "items": { "type": "object", "properties": { - "values": { - "type": "object", - "properties": { - "denorm_key": { - "type": "string", - "minLength": 1 - }, - "denorm_out_field": { - "type": "string", - "minLength": 1 - } - }, - "required": ["denorm_out_field"] + "dataset_id": { + "type": "string", + "minLength": 1 }, - "action": { + "denorm_key": { "type": "string", - "enum": ["add", "remove"] - } - }, - "additionalProperties": false, - "required": ["values", "action"], - "if": { - "properties": { - "action": { - "const": "add" - } + "minLength": 1 + }, + "denorm_out_field": { + "type": "string", + "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 } }, - "then": { - "properties": { - "values": { - "required": ["denorm_key"] - } + "oneOf": [ + { + "required": ["dataset_id", "denorm_out_field", "denorm_key"] + }, + { + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] } - } + ], + "additionalProperties": false } } + }, + "required": ["denorm_fields"], + "additionalProperties": false + }, + "connectors_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "type": "object" + }, + "operations_config": { + "type": "object" + }, + "version": { + "type": "string" + } + }, + "additionalProperties": false, + "required": ["id", "connector_id", "connector_config", "version"] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 } + }, + "sample_data": { + "type": "object" } }, - "required": ["dataset_id", "version_key"] + "required": ["dataset_id", "name"], + "additionalProperties": false } - } -} + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetExport/DatasetExport.ts b/api-service/src/controllers/DatasetExport/DatasetExport.ts new file mode 100644 index 00000000..3e118639 --- /dev/null +++ b/api-service/src/controllers/DatasetExport/DatasetExport.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import { DatasetStatus } from "../../types/DatasetModels"; +import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import _ from "lodash"; +import { obsrvError } from "../../types/ObsrvError"; + +const validateDataset = async (req: Request) => { + + const { dataset_id } = req.params; + const { status = DatasetStatus.Live } = req.query; + + let datasetRecord = status == DatasetStatus.Live ? await getLiveDatasetConfigs(dataset_id) : await datasetService.getDraftDataset(dataset_id) + if (_.isEmpty(datasetRecord)) { + throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found to export`, "NOT_FOUND", 404); + } + + const datasetStatus = _.get(datasetRecord, "status") + if (_.includes([DatasetStatus.Draft, DatasetStatus.Retired, DatasetStatus.Archived], datasetStatus)) { + throw obsrvError(dataset_id, "DATASET_EXPORT_FAILURE", `Dataset with status:${datasetStatus} cannot be exported`, "BAD_REQUEST", 400); + } + + if (_.get(datasetRecord, "api_version") != "v2") { + const migratedConfigs = await datasetService.migrateDatasetV1(dataset_id, datasetRecord) + datasetRecord = { ...datasetRecord, ...migratedConfigs } + } + return datasetRecord; +} + +const datasetExport = async (req: Request, res: Response) => { + + const datasetRecord = await validateDataset(req) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: datasetRecord }); + +} + +export default datasetExport; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetHealth/DatasetHealth.ts b/api-service/src/controllers/DatasetHealth/DatasetHealth.ts new file mode 100644 index 00000000..d523af1a --- /dev/null +++ b/api-service/src/controllers/DatasetHealth/DatasetHealth.ts @@ -0,0 +1,49 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasetHealthRequestSchema from "./DatasetHealthValidationSchema.json" +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { DatasetStatus } from "../../types/DatasetModels"; +import { getDatasetHealth, getInfraHealth } from "../../services/DatasetHealthService"; +import { obsrvError } from "../../types/ObsrvError"; +import httpStatus from "http-status"; +import { datasetService } from "../../services/DatasetService"; +export const apiId = "api.dataset.health"; + +const validateRequest = async (req: Request) => { + const isRequestValid: Record = schemaValidation(req.body, DatasetHealthRequestSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_HEALTH_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } +} + +const datasetHealth = async (req: Request, res: Response) => { + const dataset_id = _.get(req, ["body", "request", "dataset_id"]); + const categories = _.get(req, ["body", "request", "categories"]) + + validateRequest(req) + if (dataset_id === "*") { + const { components, status } = await getInfraHealth(false) + return ResponseHandler.successResponse(req, res, { + status: httpStatus.OK, data: { + status, + details: [ + { + category: "infra", + status, + components + }] + } + }); + } + const dataset = await datasetService.findDatasets({ id: dataset_id, status: DatasetStatus.Live }, ["dataset_id", "status", "type"]) + if (_.isEmpty(dataset)) { + throw obsrvError(dataset, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset} not found`, "NOT_FOUND", 404); + } + const datasetHealthData = await getDatasetHealth(categories, dataset[0]) + return ResponseHandler.successResponse(req, res, { + status: httpStatus.OK, data: datasetHealthData + }); +} + +export default datasetHealth; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetHealth/DatasetHealthValidationSchema.json b/api-service/src/controllers/DatasetHealth/DatasetHealthValidationSchema.json new file mode 100644 index 00000000..82c5a3ee --- /dev/null +++ b/api-service/src/controllers/DatasetHealth/DatasetHealthValidationSchema.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "categories": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "infra", + "processing", + "query" + ], + "required": [ + "category" + ] + } + } + }, + "required": [ + "categories", "dataset_id" + ] + } + } +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts new file mode 100644 index 00000000..e390d8bd --- /dev/null +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -0,0 +1,66 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import _ from "lodash"; +import { datasetService } from "../../services/DatasetService"; +import { datasetImportValidation, migrateExportedDatasetV1 } from "./DatasetImportHelper"; +import { obsrvError } from "../../types/ObsrvError"; + +const datasetImport = async (req: Request, res: Response) => { + + const { overwrite = "true" } = req.query; + const requestBody = req.body + let datasetPayload = requestBody.request; + if (_.get(datasetPayload, "api_version") !== "v2") { + const migratedConfigs = migrateExportedDatasetV1(requestBody) + datasetPayload = migratedConfigs; + } + const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) + const { successMsg, partialIgnored } = getResponseData(ignoredFields) + + const dataset = await importDataset(updatedDataset, overwrite); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); +} + +const importDataset = async (dataset: Record, overwrite: string | any) => { + const dataset_id = _.get(dataset,"dataset_id") + const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) + if (response?.name === "SequelizeUniqueConstraintError") { + if (overwrite === "true") { + const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ + throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); + }) + return _.omit(overwriteRes, ["message"]) + } else { + throw obsrvError(dataset_id, "DATASET_EXISTS", `Dataset with dataset_id: ${dataset_id} already exists.`, "CONFLICT", 409); + } + } + if(response?.errors){ + throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id}`, "INTERNAL_SERVER_ERROR", 500); + } + return response +} + +const getResponseData = (ignoredConfigs: Record) => { + const { ignoredConnectors, ignoredTransformations, ignoredDenorms } = ignoredConfigs; + let successMsg = "Dataset is imported successfully"; + const partialIgnored: Record = {}; + + if (ignoredConnectors.length || ignoredTransformations.length || ignoredDenorms.length) { + successMsg = "Dataset is partially imported"; + + if (ignoredTransformations.length) { + partialIgnored.transformations = ignoredTransformations; + } + if (ignoredConnectors.length) { + partialIgnored.connectors = ignoredConnectors; + } + if (ignoredDenorms.length) { + partialIgnored.denorm_fields = ignoredDenorms; + } + } + + return { successMsg, partialIgnored }; +} + +export default datasetImport; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts new file mode 100644 index 00000000..7d06a196 --- /dev/null +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -0,0 +1,154 @@ +import _ from "lodash"; +import { obsrvError } from "../../types/ObsrvError"; +import ValidationSchemaV2 from "./RequestValidationSchemaV2.json" +import ValidationSchemaV1 from "./RequestValidationSchemaV1.json" +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { schemaValidation } from "../../services/ValidationService"; +import { DatasetStatus, DatasetType } from "../../types/DatasetModels"; +import { datasetService } from "../../services/DatasetService"; + +const reqBodySchema = ValidationSchemaV2.request_body +const transformationSchema = ValidationSchemaV2.transformations_config +const connectorSchema = ValidationSchemaV2.connectors_config +const denormSchema = ValidationSchemaV2.denorm_config + +const validateConfigs = (schema: any, configs: any[]): { valid: any[], ignored: any[] } => { + const validConfigs: any[] = []; + const ignoredConfigs: any[] = []; + + for (const config of configs) { + const isConfig = schemaValidation(config, schema) + if (isConfig.isValid) { + validConfigs.push(config); + } else { + ignoredConfigs.push({ config, details: isConfig.message }); + } + } + + return { valid: validConfigs, ignored: ignoredConfigs }; +}; + +export const datasetImportValidation = async (payload: Record): Promise> => { + const isRequestValid: Record = schemaValidation(payload, reqBodySchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_IMPORT_INVALID_CONFIGS", isRequestValid.message, "BAD_REQUEST", 400) + } + + const datasetConfig = payload.request; + + const connectors = _.get(datasetConfig, "connectors_config", []); + const transformations = _.get(datasetConfig, "transformations_config", []); + const denormConfig = _.get(datasetConfig, "denorm_config", { denorm_fields: [] }); + const { validDenorms, invalidDenorms } = await validateDenorms(denormConfig) + + const { valid: resultantConnectors, ignored: ignoredConnectors } = validateConfigs(connectorSchema, connectors); + const { valid: resultantTransformations, ignored: ignoredTransformations } = validateConfigs(transformationSchema, transformations); + const { valid: resultantDenorms, ignored: ignoredDenorms } = validateConfigs(denormSchema, validDenorms); + + datasetConfig["connectors_config"] = resultantConnectors; + datasetConfig["transformations_config"] = resultantTransformations; + datasetConfig["denorm_config"] = { ...denormConfig, denorm_fields: resultantDenorms }; + datasetConfig["router_config"] = { topic: datasetConfig.id } + datasetConfig["version_key"] = Date.now().toString() + datasetConfig["status"] = DatasetStatus.Draft + + const defaults = _.cloneDeep(defaultDatasetConfig); + const resultantDataset = _.merge(defaults, datasetConfig); + + return { + updatedDataset: _.omit(resultantDataset, ["created_date", "updated_date", "published_date"]), + ignoredFields: { ignoredConnectors, ignoredTransformations, ignoredDenorms: [...ignoredDenorms, ...invalidDenorms] } + }; +}; + +const validateDenorms = async (denormConfig: Record): Promise> => { + const invalidDenorms: any[] = []; + const validDenorms: any[] = []; + + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: DatasetType.master }, ["id", "dataset_id", "status", "dataset_config", "api_version"]); + + for (const field of denormConfig.denorm_fields) { + const { redis_db, dataset_id, denorm_out_field, denorm_key } = field; + let masterDataset; + + if (dataset_id) { + masterDataset = _.find(masterDatasets, dataset => _.get(dataset, "dataset_id") === dataset_id); + } else if (redis_db) { + masterDataset = _.find(masterDatasets, dataset => _.get(dataset, "dataset_config.cache_config.redis_db") === redis_db); + } + + const denormFields = { denorm_key, denorm_out_field, dataset_id: dataset_id || _.get(masterDataset, "dataset_id") } + if (masterDataset) { + validDenorms.push(denormFields); + } else { + invalidDenorms.push({ config: denormFields, details: `Master dataset does not exist` }); + } + } + } + + return { validDenorms, invalidDenorms }; +}; + +export const migrateExportedDatasetV1 = (requestPayload: Record) => { + + const v1Config = schemaValidation(requestPayload, ValidationSchemaV1); + if (!v1Config.isValid) { + throw obsrvError("", "DATASET_V1_CONFIGS_INVALID", v1Config.message, "BAD_REQUEST", 400) + } + const datasetPayload = requestPayload.request + const { dataset_id, timestamp_key = "", data_key = "", type: datasetType } = _.get(datasetPayload, "data.metadata") + const type = datasetType === "master-dataset" ? DatasetType.master : DatasetType.event + + const dataset: Record = { + dataset_id, id: dataset_id, name: dataset_id, type, + version_key: Date.now().toString(), + api_version: "v2", + }; + + const { validation, dedup, batch } = _.get(datasetPayload, "data.config") + dataset["data_schema"] = _.get(datasetPayload, "data.data_schema") + dataset["dedup_config"] = { ..._.omit(dedup, "enabled"), drop_duplicates: _.get(dedup, "enabled") }; + dataset["router_config"] = { topic: dataset_id }; + dataset["validation_config"] = { ..._.omit(validation, "enabled"), validate: _.get(validation, "enabled") }; + + const { drop_duplicates, dedup_key, dedup_period, extraction_key, enabled: is_batch_event } = batch + dataset["extraction_config"] = { is_batch_event, extraction_key, dedup_config: { drop_duplicates, dedup_key, dedup_period } } + + const { redis_db, redis_db_host, redis_db_port } = defaultDatasetConfig.dataset_config.cache_config; + dataset["dataset_config"] = { + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (type === DatasetType.master) }, + keys_config: { data_key, timestamp_key }, + cache_config: { redis_db_host, redis_db_port, redis_db } + } + + dataset["denorm_config"] = { + denorm_fields: _.map(_.get(datasetPayload, "data.denorm"), configs => { + const { master_dataset_id, denorm_key, out } = configs; + return { denorm_key, denorm_out_field: out, dataset_id: master_dataset_id } + }) + } + + dataset["transformations_config"] = _.map(_.get(datasetPayload, "data.transformations", []), (config) => { + const { type, key, expr, mode, dataType = "string", section } = config + return { + field_key: key, + transformation_function: { + type, expr, + datatype: dataType, + category: datasetService.getTransformationCategory(section) + }, mode + } + }) + + dataset["connectors_config"] = _.map(_.get(datasetPayload, "env_variables.input_sources", []), (config) => { + const { id, type, ...rest } = config + return { + id, connector_id: type, + connector_config: rest, + version: "v1" + } + }) + + return dataset; +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json new file mode 100644 index 00000000..41acd1f2 --- /dev/null +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json @@ -0,0 +1,234 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.import"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "version": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "env_variables": { + "type": "object", + "properties": { + "input_sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": ["id", "type"] + } + } + }, + "required": ["input_sources"] + }, + "data": { + "type": "object", + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": ["dataset", "master-dataset"] + }, + "timestamp_key": { + "type": "string", + "minLength": 1 + }, + "data_key": { + "type": "string", + "minLength": 1 + } + }, + "required": ["id", "dataset_id", "type"] + }, + "data_schema": { + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "type": { + "type": "string" + }, + "properties": { + "type": "object" + }, + "additionalProperties": { + "type": "boolean" + } + }, + "required": ["$schema", "type", "properties"] + }, + "config": { + "type": "object", + "properties": { + "batch": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "extraction_key": { + "type": "string" + }, + "drop_duplicates": { + "type": "boolean" + }, + "dedup_period": { + "type": "integer" + }, + "dedup_key": { + "type": "string" + } + }, + "required": [ + "enabled", + "id", + "extraction_key", + "drop_duplicates", + "dedup_key" + ] + }, + "dedup": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "required": ["enabled", "dedup_key"] + }, + "router": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "minLength": 1 + } + } + }, + "validation": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "mode": { + "type": "string" + } + }, + "required": ["enabled", "mode"] + } + }, + "required": ["batch", "dedup", "validation"] + }, + "transformations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "key": { + "type": "string" + }, + "expr": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "dataType": { + "type": "string" + }, + "section": { + "type": "string" + } + }, + "required": [ + "type", + "key", + "expr", + "mode", + "dataType", + "section" + ] + } + }, + "denorm": { + "type": "array", + "items": { + "type": "object", + "properties": { + "master_dataset_id": { + "type": "string" + }, + "denorm_key": { + "type": "string" + }, + "out": { + "type": "string" + } + }, + "required": ["master_dataset_id", "denorm_key", "out"] + } + } + }, + "required": ["metadata", "data_schema"] + } + }, + "required": ["data"] + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json new file mode 100644 index 00000000..da61770c --- /dev/null +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -0,0 +1,456 @@ +{ + "request_body": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.import"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": ["event", "transaction", "master"] + }, + "version": { + "type": "integer" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "validation_config": { + "type": "object", + "properties": { + "validate": { + "type": "boolean", + "default": true + }, + "mode": { + "type": "string", + "enum": ["Strict", "IgnoreNewFields"], + "default": "Strict" + } + }, + "required": ["validate"], + "additionalProperties": false, + "if": { + "properties": { + "validate": { + "const": true + } + } + }, + "then": { + "required": ["mode"] + } + }, + "extraction_config": { + "type": "object", + "properties": { + "is_batch_event": { + "type": "boolean", + "default": true + }, + "batch_id": { + "type": "string" + }, + "extraction_key": { + "type": "string", + "default": "events" + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean", + "default": false + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": ["is_batch_event"], + "if": { + "properties": { + "is_batch_event": { + "const": true + } + } + }, + "then": { + "properties": { + "extraction_key": { + "minLength": 1 + } + }, + "required": ["extraction_key", "dedup_config"] + } + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean", + "default": true + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], + "additionalProperties": false + }, + "data_schema": { + "type": "object", + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "minLength": 1 + }, + "properties": { + "type": "object", + "minProperties": 1 + }, + "additionalProperties": { + "type": "boolean", + "default": true + }, + "required": { + "type": "array" + } + }, + "required": ["$schema", "type", "properties"], + "additionalProperties": false + }, + "dataset_config": { + "type": "object", + "properties": { + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "cache_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string", + "minLength": 1 + }, + "redis_db_port": { + "type": "integer" + }, + "redis_db": { + "type": "integer" + } + }, + "required": ["redis_db_host", "redis_db_port"], + "additionalProperties": false + }, + "file_upload_path": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["indexing_config", "keys_config"], + "additionalProperties": false + }, + "transformations_config": { + "type": "array" + }, + "connectors_config": { + "type": "array" + }, + "denorm_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string" + }, + "redis_db_port": { + "type": "integer" + }, + "denorm_fields": { + "type": "array" + } + }, + "required": ["denorm_fields"], + "additionalProperties": false + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "router_config": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "minLength": 1 + } + }, + "required": ["topic"], + "additionalProperties": false + }, + "sample_data": { + "type": "object" + }, + "entry_topic": { + "type": "string", + "minLength": 1 + }, + "data_version": { + "type": "integer" + }, + "api_version": { + "type": "string", + "minLength": 1 + }, + "version_key": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string" + }, + "created_date": { + "type": "string" + }, + "updated_date": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "updated_by": { + "type": "string" + } + }, + "required": [ + "id", + "dataset_id", + "name", + "type", + "data_schema", + "dataset_config" + ], + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false + }, + "transformations_config": { + "type": "object", + "properties": { + "field_key": { + "type": "string", + "minLength": 1 + }, + "transformation_function": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string" + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "mode": { + "type": "string", + "enum": ["Strict", "Lenient"] + } + }, + "additionalProperties": false, + "required": ["field_key", "transformation_function", "mode"] + }, + "connectors_config": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "type": "object" + }, + "operations_config": { + "type": "object" + }, + "version": { + "type": "string" + } + }, + "additionalProperties": false, + "required": ["id", "connector_id", "connector_config"] + }, + "denorm_config": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "denorm_key": { + "type": "string", + "minLength": 1 + }, + "denorm_out_field": { + "type": "string", + "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 + } + }, + "oneOf": [ + { + "required": ["dataset_id", "denorm_out_field", "denorm_key"] + }, + { + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + } + ] + } +} diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts new file mode 100644 index 00000000..82acbf62 --- /dev/null +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -0,0 +1,44 @@ +import _ from "lodash"; +import httpStatus from "http-status"; +import { Request, Response } from "express"; +import logger from "../../logger"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasetCreate from "./DatasetListValidationSchema.json"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { datasetService } from "../../services/DatasetService"; +import { obsrvError } from "../../types/ObsrvError"; + +export const apiId = "api.datasets.list" +export const errorCode = "DATASET_LIST_FAILURE" +const liveDatasetStatus = ["Live", "Retired", "Purged"] +const draftDatasetStatus = ["Draft", "ReadyToPublish"] +const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date"] + +const datasetList = async (req: Request, res: Response) => { + + const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) + } + + const datasetBody = req.body.request; + const datasetList = await listDatasets(datasetBody) + const responseData = { data: datasetList, count: _.size(datasetList) } + logger.info({req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); + +} + +const listDatasets = async (request: Record): Promise> => { + + const { filters = {} } = request || {}; + const datasetStatus = _.get(filters, "status"); + const status = _.isArray(datasetStatus) ? datasetStatus : _.compact([datasetStatus]) + const draftFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasetStatus : _.intersection(status, draftDatasetStatus)); + const liveFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasetStatus : _.intersection(status, liveDatasetStatus)); + const liveDatasetList = await datasetService.findDatasets(liveFilters, defaultFields, [["updated_date", "DESC"]]); + const draftDatasetList = await datasetService.findDraftDatasets(draftFilters, defaultFields, [["updated_date", "DESC"]]); + return _.compact(_.concat(liveDatasetList, draftDatasetList)); +} + +export default datasetList; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json b/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json similarity index 61% rename from api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json rename to api-service/src/controllers/DatasetList/DatasetListValidationSchema.json index f6a6955e..3435d631 100644 --- a/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json +++ b/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json @@ -33,7 +33,7 @@ "type": "array", "items": { "type": "string", - "enum": ["Draft", "Live", "ReadyToPublish", "Retired"] + "enum": ["Draft", "Live", "ReadyToPublish", "Retired", "Purged"] } }, { @@ -44,37 +44,10 @@ }, "type": { "type": "string", - "enum": ["dataset", "master-dataset"] - }, - "dataset_id": { - "type": "string", - "minLength": 1 + "enum": ["event", "transaction", "master"] } }, "additionalProperties": false - }, - "sortBy": { - "type": "array", - "items": { - "type": "object", - "properties": { - "field": { - "type": "string", - "enum": [ - "created_date", - "updated_date", - "published_date", - "name" - ] - }, - "order": { - "type": "string", - "enum": ["asc", "desc"] - } - }, - "additionalProperties": false, - "required": ["field", "order"] - } } }, "additionalProperties": false diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts new file mode 100644 index 00000000..903d393c --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DatasetMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") +} + +const datasetMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, "data"); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json b/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json new file mode 100644 index 00000000..ae17e302 --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "query": { + "type": "object", + "nullable": true + } + }, + "required": [ + "query" + ] +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts new file mode 100644 index 00000000..e1853e57 --- /dev/null +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -0,0 +1,114 @@ +import { Request, Response } from "express"; +import httpStatus from "http-status"; +import _ from "lodash"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { DatasetDraft } from "../../models/DatasetDraft"; +import { datasetService, getV1Connectors } from "../../services/DatasetService"; +import { obsrvError } from "../../types/ObsrvError"; +import { cipherService } from "../../services/CipherService"; + +export const apiId = "api.datasets.read"; +export const errorCode = "DATASET_READ_FAILURE" + +// TODO: Move this to a config +export const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] + +const validateRequest = (req: Request) => { + + const { dataset_id } = req.params; + const { fields } = req.query; + const fieldValues = fields ? _.split(fields as string, ",") : []; + const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())); + if (!_.isEmpty(invalidFields)) { + throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS", `The specified fields [${invalidFields}] in the dataset cannot be found.`, "BAD_REQUEST", 400); + } + +} + +const datasetRead = async (req: Request, res: Response) => { + + validateRequest(req); + const { dataset_id } = req.params; + const { fields, mode } = req.query; + const attributes = !fields ? defaultFields : _.split(fields, ","); + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) + if (!dataset) { + throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); + } + if (dataset.connectors_config) { + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); + } + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); +} + +const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { + + const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) + const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); + if (draftDataset) { // Contains a draft + const apiVersion = _.get(draftDataset, ["api_version"]); + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) + return _.pick(dataset, attributes); + } + + const liveDataset = await datasetService.getDataset(datasetId, undefined, true); + if (liveDataset) { + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) + return _.pick(dataset, attributes); + } + + return null; +} + +const readDataset = async (datasetId: string, attributes: string[]): Promise => { + const dataset = await datasetService.getDataset(datasetId, attributes, true); + if (!dataset) { + return; + } + const api_version = _.get(dataset, "api_version") + const datasetConfigs: any = {} + const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) + if (api_version !== "v2") { + datasetConfigs["connectors_config"] = await getV1Connectors(datasetId) + datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { + const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); + return { + field_key: _.get(config, "field_key"), + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, "metadata._transformedFieldDataType") || _.get(config, "transformation_function.datatype") || "string", + category: datasetService.getTransformationCategory(section), + }, + mode: _.get(config, "mode") + } + }) + } + else { + const v1connectors = await getV1Connectors(datasetId) + const v2connectors = await datasetService.getConnectors(datasetId, ["id", "connector_id", "connector_config", "operations_config"]); + datasetConfigs["connectors_config"] = _.concat(v1connectors, v2connectors) + datasetConfigs["transformations_config"] = transformations_config; + } + return { ...dataset, ...datasetConfigs }; +} + +const processConnectorsConfig = (connectorsConfig: any) => { + return connectorsConfig.map((connector: any) => { + let connector_config = _.get(connector, "connector_config"); + const authMechanism = _.get(connector_config, ["authenticationMechanism"]); + + if (authMechanism && authMechanism.encrypted) { + connector_config = { + ...connector_config, + authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues)) + }; + } + + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) + }; + }); +} + +export default datasetRead; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts new file mode 100644 index 00000000..68b272f1 --- /dev/null +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -0,0 +1,57 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasetResetRequestSchema from "./DatasetResetValidationSchema.json" +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { HealthStatus } from "../../types/DatasetModels"; +import { getDruidIndexers, getFlinkHealthStatus, restartDruidIndexers } from "../../services/DatasetHealthService"; +import { restartPipeline } from "../DatasetStatusTransition/DatasetStatusTransition"; +import { obsrvError } from "../../types/ObsrvError"; +import { datasetService } from "../../services/DatasetService"; +import httpStatus from "http-status"; + +export const apiId = "api.dataset.reset"; + +const validateRequest = async (req: Request) => { + + const isRequestValid: Record = schemaValidation(req.body, DatasetResetRequestSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } + const datasetId = _.get(req, ["params", "datasetId"]) + const isDataSetExists = await datasetService.checkDatasetExists(datasetId); + if (!isDataSetExists) { + throw obsrvError(datasetId, "DATASET_NOT_FOUND", `Dataset not exists with id:${datasetId}`, httpStatus[httpStatus.NOT_FOUND], 404) + } +} + +const datasetReset = async (req: Request, res: Response) => { + + const category = _.get(req, ["body", "request", "category"]); + const datasetId = _.get(req, ["params"," datasetId"]); + + await validateRequest(req); + if (category == "processing") { + const pipeLineStatus = await getFlinkHealthStatus() + if (pipeLineStatus == HealthStatus.UnHealthy) { + await restartPipeline({ "dataset": { "dataset_id": datasetId } }) + } + } else if (category == "query") { + const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) + if(!_.isEmpty(datasources)) { + const unHealthySupervisors = await getDruidIndexers(datasources, HealthStatus.UnHealthy) + const unHealthyDataSources = _.filter(unHealthySupervisors, (supervisor: any) => supervisor?.state == "SUSPENDED") + if (!_.isEmpty(unHealthyDataSources)) { + await restartDruidIndexers(unHealthyDataSources) + } + } + } + + return ResponseHandler.successResponse(req, res, { + status: 200, data: { + "status": "Completed" + } + }); +} + +export default datasetReset; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetReset/DatasetResetValidationSchema.json b/api-service/src/controllers/DatasetReset/DatasetResetValidationSchema.json new file mode 100644 index 00000000..9450c235 --- /dev/null +++ b/api-service/src/controllers/DatasetReset/DatasetResetValidationSchema.json @@ -0,0 +1,38 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "category": { + "type": "string", + "enum": ["processing", "query"] + } + }, + "required": [ + "category" + ] + } + } +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts new file mode 100644 index 00000000..c2a18021 --- /dev/null +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -0,0 +1,252 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { datasetService } from "../../services/DatasetService"; +import { schemaValidation } from "../../services/ValidationService"; +import StatusTransitionSchema from "./RequestValidationSchema.json"; +import ReadyToPublishSchema from "./ReadyToPublishSchema.json" +import httpStatus from "http-status"; +import { DatasetStatus, DatasetType } from "../../types/DatasetModels"; +import { executeCommand } from "../../connections/commandServiceConnection"; +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { obsrvError } from "../../types/ObsrvError"; + +const invalidRequest = "DATASET_STATUS_TRANSITION_INVALID_INPUT" +const datasetNotFound = "DATASET_NOT_FOUND" + +const allowedTransitions: Record = { + Delete: [DatasetStatus.Draft, DatasetStatus.ReadyToPublish], + ReadyToPublish: [DatasetStatus.Draft], + Live: [DatasetStatus.ReadyToPublish], + Retire: [DatasetStatus.Live], + Archive: [DatasetStatus.Retired], + Purge: [DatasetStatus.Archived] +} +const liveDatasetActions = ["Retire", "Archive", "Purge"] + +const validateRequest = (req: Request, datasetId: any) => { + const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) + if (!isRequestValid.isValid) { + throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) + } +} + +const validateDataset = (dataset: any, datasetId: any, action: string) => { + + if (_.isEmpty(dataset)) { + throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${datasetId}`, "NOT_FOUND", 404) + } + + if (dataset.api_version !== "v2" && _.includes(["ReadyToPublish", "Live"], action)) { + throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "BAD_REQUEST", 400) + } + + if (!_.includes(allowedTransitions[action], dataset.status)) { + throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "CONFLICT", 409) + } + + return true; +} + + +const datasetStatusTransition = async (req: Request, res: Response) => { + + const { dataset_id, status } = _.get(req.body, "request"); + validateRequest(req, dataset_id); + + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + validateDataset(dataset, dataset_id, status); + + switch (status) { + case "Delete": + await deleteDataset(dataset); + break; + case "ReadyToPublish": + await readyForPublish(dataset); + break; + case "Live": + await publishDataset(dataset); + break; + case "Retire": + await retireDataset(dataset); + break; + case "Archive": + await archiveDataset(dataset); + break; + case "Purge": + await purgeDataset(dataset); + break; + } + + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); +} + + +// Delete a draft dataset +const deleteDataset = async (dataset: Record) => { + + await datasetService.deleteDraftDataset(dataset) + // TODO: Delete any sample files or schemas that are uploaded +} + + +const readyForPublish = async (dataset: Record) => { + + const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) + defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); + if (draftDataset?.type === "master") { + defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); + } + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { + if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { + return objValue; + } + }); + const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) + if (!datasetValid.isValid) { + throw { + code: "DATASET_CONFIGS_INVALID", + message: datasetValid.message, + errCode: "BAD_REQUEST", + statusCode: 400 + } + } + _.set(draftDataset, "status", DatasetStatus.ReadyToPublish) + await datasetService.updateDraftDataset(draftDataset) +} + +/** + * Method to publish a dataset. Does the following: + * 1. Validate if all the denorm datasets are valid, no cirular reference and are in Live status + * 2. Update the redis host and db if the dataset is a master dataset + * 3. Save the draft copy + * 4. Generate the Druid and Hudi datasource configuration depending on the storage configured + * + * @param dataset + */ +const publishDataset = async (dataset: Record) => { + + const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record + + await validateAndUpdateDenormConfig(draftDataset); + await updateMasterDataConfig(draftDataset) + await datasetService.publishDataset(draftDataset) +} + +const validateAndUpdateDenormConfig = async (draftDataset: Record) => { + + // 1. Check if there are denorm fields and dependent master datasets are published + const denormConfig = _.get(draftDataset, "denorm_config") + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + const datasetIds = _.map(denormConfig.denorm_fields, "dataset_id") + if (_.includes(datasetIds, draftDataset.id)) { + throw { + code: "SELF_REFERENCING_MASTER_DATA", + message: `The denorm master dataset is self-referencing itself`, + errCode: "CONFLICT", + statusCode: 409 + } + } + const masterDatasets = await datasetService.findDatasets({ id: datasetIds, type: "master" }, ["id", "status", "dataset_config", "api_version"]) + const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { + const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) + const datasetStatus: Record = { + dataset_id: denormField.dataset_id, + exists: (md) ? true : false, + isLive: (md) ? md.status === "Live" : false, + status: (md) ? md.status : false + } + if (!_.isEmpty(md)) { + if (md.api_version === "v2") + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.cache_config.redis_db }); + else + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.redis_db }); + } + + return datasetStatus; + }) + const invalidMasters = _.filter(masterDatasetsStatus, { isLive: false }) + if (_.size(invalidMasters) > 0) { + const invalidIds = _.map(invalidMasters, "dataset_id") + throw { + code: "DEPENDENT_MASTER_DATA_NOT_LIVE", + message: `The datasets with id:${invalidIds} are not in published status`, + errCode: "PRECONDITION_REQUIRED", + statusCode: 428 + } + } + + // 2. Populate redis db for denorm + draftDataset["denorm_config"] = { + redis_db_host: defaultDatasetConfig.denorm_config.redis_db_host, + redis_db_port: defaultDatasetConfig.denorm_config.redis_db_port, + denorm_fields: _.map(masterDatasetsStatus, "denorm_field") + } + } +} + +const updateMasterDataConfig = async (draftDataset: Record) => { + if (draftDataset.type === "master") { + const dataset_config = _.get(draftDataset, "dataset_config") + const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") + draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } + if (draftDataset.dataset_config.cache_config.redis_db === 0) { + const { results }: any = await datasetService.getNextRedisDBIndex() + if (_.isEmpty(results)) { + throw { + code: "REDIS_DB_INDEX_FETCH_FAILED", + message: `Unable to fetch the redis db index for the master data`, + errCode: "INTERNAL_SERVER_ERROR", + statusCode: 500 + } + } + const nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; + _.set(draftDataset, "dataset_config.cache_config.redis_db", nextRedisDB) + } + } +} + +const retireDataset = async (dataset: Record) => { + + await canRetireIfMasterDataset(dataset); + await datasetService.retireDataset(dataset); + await restartPipeline(dataset); +} + + +const canRetireIfMasterDataset = async (dataset: Record) => { + + if (dataset.type === DatasetType.master) { + + const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["dataset_config", "dataset_id"]) || [] + const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] + const allDatasets = _.union(liveDatasets, draftDatasets) + const extractDenormFields = _.map(allDatasets, function (depDataset) { + return { dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id") } + }) + const deps = _.filter(extractDenormFields, function (depDS) { return _.includes(depDS.denorm_datasets, dataset.id) }) + if (_.size(deps) > 0) { + + const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function (o) { return _.omit(o, "denorm_datasets") })) + } + } +} + +export const restartPipeline = async (dataset: Record) => { + return executeCommand(dataset.id, "RESTART_PIPELINE") +} + +const archiveDataset = async (dataset: Record) => { + + throw obsrvError(dataset.id, "ARCHIVE_NOT_IMPLEMENTED", "Archive functionality is not implemented", "NOT_IMPLEMENTED", 501) +} + +const purgeDataset = async (dataset: Record) => { + + throw obsrvError(dataset.id, "PURGE_NOT_IMPLEMENTED", "Purge functionality is not implemented", "NOT_IMPLEMENTED", 501) +} + +export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json new file mode 100644 index 00000000..eff208cb --- /dev/null +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -0,0 +1,518 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer" + }, + "status": { + "type": "string", + "enum": [ + "Draft" + ] + }, + "type": { + "type": "string", + "enum": [ + "event", + "transaction", + "master" + ] + }, + "name": { + "type": "string", + "minLength": 1 + }, + "version_key": { + "type": "string", + "minLength": 1 + }, + "validation_config": { + "type": "object", + "properties": { + "validate": { + "type": "boolean" + }, + "mode": { + "type": "string", + "enum": [ + "Strict", + "IgnoreNewFields" + ] + } + }, + "required": [ + "validate", + "mode" + ] + }, + "extraction_config": { + "type": "object", + "properties": { + "is_batch_event": { + "type": "boolean" + }, + "extraction_key": { + "type": "string" + }, + "batch_id": { + "type": "string" + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean" + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": [ + "dedup_key", + "dedup_period" + ] + }, + "required": [ + "drop_duplicates" + ], + "additionalProperties": false + } + }, + "if": { + "properties": { + "is_batch_event": { + "const": true + } + } + }, + "then": { + "properties": { + "extraction_key": { + "minLength": 1 + }, + "batch_id": { + "minLength": 1 + } + } + }, + "required": [ + "is_batch_event", + "extraction_key", + "dedup_config" + ], + "additionalProperties": false + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean" + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": [ + "dedup_key", + "dedup_period" + ] + }, + "required": [ + "drop_duplicates" + ], + "additionalProperties": false + }, + "data_schema": { + "type": "object", + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "minLength": 1 + }, + "properties": { + "type": "object", + "minProperties": 1 + }, + "required": { + "type": "array" + }, + "additionalProperties": { + "type": "boolean" + } + }, + "required": [ + "type", + "properties" + ], + "additionalProperties": false + }, + "denorm_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string", + "minLength": 1 + }, + "redis_db_port": { + "type": "integer" + }, + "denorm_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "denorm_key": { + "type": "string", + "minLength": 1 + }, + "denorm_out_field": { + "type": "string", + "minLength": 1 + }, + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 + }, + "redis_db": { + "type": "integer" + } + }, + "oneOf": [ + { + "required": [ + "dataset_id", + "denorm_out_field", + "denorm_key" + ] + }, + { + "required": [ + "dataset_id", + "denorm_out_field", + "jsonata_expr" + ] + } + ], + "additionalProperties": false + } + } + }, + "required": [ + "denorm_fields" + ], + "additionalProperties": false + }, + "router_config": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "topic" + ], + "additionalProperties": false + }, + "dataset_config": { + "type": "object", + "properties": { + "file_upload_path": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "dataset_tz": { + "type": "string" + }, + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "required": [ + "olap_store_enabled", + "lakehouse_enabled", + "cache_enabled" + ], + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "data_key", + "timestamp_key" + ], + "additionalProperties": false + }, + "cache_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string", + "minLength": 1 + }, + "redis_db_port": { + "type": "integer" + }, + "redis_db": { + "type": "integer" + } + }, + "required": [ + "redis_db_host", + "redis_db_port" + ], + "additionalProperties": false + } + }, + "required": [ + "indexing_config", + "keys_config", + "cache_config" + ], + "additionalProperties": false + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "sample_data": { + "type": "object" + }, + "transformations_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field_key": { + "type": "string", + "minLength": 1 + }, + "transformation_function": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string" + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string" + } + }, + "required": [ + "type", + "expr" + ], + "additionalProperties": false + }, + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": [ + "pii", + "transform", + "derived" + ] + } + }, + "required": [ + "type", + "expr", + "category" + ], + "additionalProperties": false + }, + "mode": { + "type": "string", + "enum": [ + "Strict", + "Lenient" + ] + } + }, + "additionalProperties": false, + "required": [ + "field_key", + "transformation_function", + "mode" + ] + } + }, + "connectors_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] + }, + "operations_config": { + "type": "object" + }, + "version": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false, + "required": [ + "id", + "connector_id", + "connector_config", + "version" + ] + } + } + }, + "if": { + "properties": { + "type": { + "const": "master" + } + } + }, + "then": { + "properties": { + "dataset_config": { + "properties": { + "keys_config": { + "properties": { + "data_key": { + "minLength": 1 + } + }, + "required": [ + "data_key" + ] + } + } + } + } + }, + "required": [ + "dataset_id", + "type", + "name", + "version", + "data_schema", + "id", + "validation_config", + "extraction_config", + "dataset_config", + "denorm_config", + "dedup_config", + "router_config", + "version_key", + "status", + "transformations_config", + "connectors_config" + ] +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetStatusTransition/RequestValidationSchema.json b/api-service/src/controllers/DatasetStatusTransition/RequestValidationSchema.json new file mode 100644 index 00000000..c7455af6 --- /dev/null +++ b/api-service/src/controllers/DatasetStatusTransition/RequestValidationSchema.json @@ -0,0 +1,52 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "api.datasets.status-transition" + ] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "ReadyToPublish", + "Retire", + "Delete", + "Live" + ] + } + }, + "additionalProperties": false, + "required": [ + "dataset_id", + "status" + ] + } + } + } \ No newline at end of file diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts new file mode 100644 index 00000000..274ad3c4 --- /dev/null +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -0,0 +1,148 @@ +import { Request, Response } from "express"; +import httpStatus from "http-status"; +import _ from "lodash"; +import Model from "sequelize/types/model"; +import { DatasetStatus } from "../../types/DatasetModels"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { cipherService } from "../../services/CipherService"; +import { datasetService } from "../../services/DatasetService"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; +import { obsrvError } from "../../types/ObsrvError"; +import logger from "../../logger"; + +export const apiId = "api.datasets.update"; +export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" +export const errorCode = "DATASET_UPDATE_FAILURE" + +const validateRequest = async (req: Request) => { + + const datasetId = _.get(req, ["body", "request", "dataset_id"]) + const isRequestValid: Record = schemaValidation(req.body, DatasetUpdate) + if (!isRequestValid.isValid) { + throw obsrvError(datasetId, "DATASET_UPDATE_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) + } + + const datasetBody = req.body.request + const { dataset_id, version_key, ...rest } = datasetBody + if (_.isEmpty(rest)) { + logger.error({ apiId, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) + throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) + } + +} + +const validateDataset = (dataset: Record | null, req: Request) => { + + const datasetId = _.get(req, ["body", "request", "dataset_id"]) + const versionKey = _.get(req, ["body", "request", "version_key"]) + if (dataset) { + if (dataset.api_version !== "v2") { + throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "NOT_FOUND", 404) + } + if(dataset.version_key !== versionKey) { + throw obsrvError(datasetId, "DATASET_OUTDATED", "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", "CONFLICT", 409) + } + if(!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { + throw obsrvError(datasetId, "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", "Dataset cannot be updated as it is not in draft state", "BAD_REQUEST", 400) + } + } else { + throw obsrvError(datasetId, "DATASET_NOT_EXISTS", `Dataset does not exists with id:${datasetId}`, "NOT_FOUND", 404) + } +} + +const datasetUpdate = async (req: Request, res: Response) => { + + await validateRequest(req) + const datasetReq = req.body.request; + const datasetModel = await datasetService.getDraftDataset(datasetReq.dataset_id) + validateDataset(datasetModel, req) + + const draftDataset = mergeDraftDataset(datasetModel, datasetReq); + const response = await datasetService.updateDraftDataset(draftDataset); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); +} + +const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { + + const dataset: Record = { + version_key: Date.now().toString(), + name: datasetReq.name || _.get(datasetModel, ["name"]), + id: _.get(datasetModel, ["id"]) + } + if(datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config + if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config + if(datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config + if(datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema + if(datasetReq.dataset_config) dataset["dataset_config"] = datasetReq.dataset_config + if(datasetReq.transformations_config) + dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config) + if(datasetReq.denorm_config) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config) + if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) + if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) + if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data + if(datasetReq.type) dataset["type"] = datasetReq.type + + return dataset; +} + +const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any) => { + const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.field_key") + const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + + return _.unionWith( + addConfigs, + _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key)}), + (a, b) => { + return a.field_key === b.field_key + } + ) +} + +const mergeDenormConfig = (currentConfig: any, newConfig: any) => { + + const removeConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "remove"}), "value.denorm_out_field") + const addConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "upsert"}), "value") + + const denormFields = _.unionWith( + addConfigs, + _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field)}), + (a, b) => { + return a.denorm_out_field === b.denorm_out_field + } + ) + return { + denorm_fields: denormFields + } +} + +const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { + + const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.connector_id") + const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + + return _.unionWith( + _.map(addConfigs, (config) => { + return { + id: config.id, + connector_id: config.connector_id, + connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), + operations_config: config.operations_config, + version: config.version + } + }), + _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id)}), + (a, b) => { + return a.connector_id === b.connector_id + } + ) +} + +const mergeTags = (currentTags: any, newConfigs: any) => { + + const tagsToRemove = _.map(_.filter(newConfigs, {action: "remove"}), "value") + const tagsToAdd = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + return _.union(_.pullAll(currentTags, tagsToRemove), tagsToAdd) +} + +export default datasetUpdate; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json new file mode 100644 index 00000000..b31fe674 --- /dev/null +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -0,0 +1,453 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.update"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "additionalProperties": false, + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "version_key": { + "type": "string" + }, + "type": { + "type": "string", + "enum": ["event", "transaction", "master"] + }, + "name": { + "type": "string", + "minLength": 1 + }, + "validation_config": { + "type": "object", + "properties": { + "validate": { + "type": "boolean" + }, + "mode": { + "type": "string", + "enum": ["Strict", "IgnoreNewFields"] + } + }, + "required": ["validate"], + "additionalProperties": false, + "if": { + "properties": { + "validate": { + "const": true + } + } + }, + "then": { + "required": ["mode"] + } + }, + "extraction_config": { + "type": "object", + "properties": { + "is_batch_event": { + "type": "boolean" + }, + "batch_id": { + "type": "string" + }, + "extraction_key": { + "type": "string" + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean" + }, + "dedup_key": { + "type": "string" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": ["is_batch_event"], + "if": { + "properties": { + "is_batch_event": { + "const": true + } + } + }, + "then": { + "required": ["extraction_key", "dedup_config"] + } + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean", + "default": true + }, + "dedup_key": { + "type": "string" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], + "additionalProperties": false + }, + "data_schema": { + "type": "object", + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "minLength": 1 + }, + "required": { + "type": "array" + }, + "properties": { + "type": "object", + "minProperties": 1 + }, + "additionalProperties": { + "type": "boolean", + "default": true + } + }, + "required": ["$schema", "type", "properties", "additionalProperties"], + "additionalProperties": false + }, + "dataset_config": { + "type": "object", + "properties": { + "file_upload_path": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "dataset_tz": { + "type": "string" + }, + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + } + }, + "required": ["indexing_config", "keys_config"], + "additionalProperties": false + }, + "transformations_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "object", + "properties": { + "field_key": { + "type": "string", + "minLength": 1 + }, + "transformation_function": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "mode": { + "type": "string", + "enum": [ + "Strict", + "Lenient" + ] + } + }, + "required": ["field_key"], + "additionalProperties": false + }, + "action": { + "type": "string", + "enum": ["upsert", "remove"] + } + }, + "additionalProperties": false, + "required": ["value", "action"], + "if": { + "properties": { + "action": { + "const": "upsert" + } + } + }, + "then": { + "properties": { + "value": { + "required": ["transformation_function", "mode"] + } + } + } + } + }, + "denorm_config": { + "type": "object", + "properties": { + "denorm_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "denorm_key": { + "type": "string", + "minLength": 1 + }, + "denorm_out_field": { + "type": "string", + "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "action": { + "type": "string", + "enum": ["upsert", "remove"] + } + }, + "additionalProperties": false, + "required": ["value", "action"], + "if": { + "properties": { + "action": { + "const": "upsert" + } + } + }, + "then": { + "properties": { + "value": { + "oneOf": [ + { + "required": ["dataset_id", "denorm_out_field", "denorm_key"] + }, + { + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + } + ] + } + } + } + } + } + } + }, + "connectors_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "type": "object" + }, + "operations_config": { + "type": "object" + }, + "version": { + "type": "string" + } + }, + "required": ["id"], + "additionalProperties": false + }, + "action": { + "type": "string", + "enum": ["upsert", "remove"] + } + }, + "additionalProperties": false, + "required": ["value", "action"], + "if": { + "properties": { + "action": { + "const": "upsert" + } + } + }, + "then": { + "properties": { + "value": { + "required": ["connector_id", "connector_config"] + } + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "string", + "minLength": 1 + }, + "action": { + "type": "string", + "enum": ["upsert", "remove"] + } + }, + "required": ["value", "action"] + } + }, + "sample_data": { + "type": "object" + } + }, + "required": ["dataset_id", "version_key"] + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} diff --git a/api-service/src/v2/controllers/DeleteQueryTemplate/DeleteTemplateController.ts b/api-service/src/controllers/DeleteQueryTemplate/DeleteTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/DeleteQueryTemplate/DeleteTemplateController.ts rename to api-service/src/controllers/DeleteQueryTemplate/DeleteTemplateController.ts diff --git a/api-service/src/v2/controllers/EventValidation/EventValidation.ts b/api-service/src/controllers/EventValidation/EventValidation.ts similarity index 91% rename from api-service/src/v2/controllers/EventValidation/EventValidation.ts rename to api-service/src/controllers/EventValidation/EventValidation.ts index 718a4785..bd850968 100644 --- a/api-service/src/v2/controllers/EventValidation/EventValidation.ts +++ b/api-service/src/controllers/EventValidation/EventValidation.ts @@ -4,7 +4,7 @@ import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import validationSchema from "./RequestValidationSchema.json" import logger from "../../logger"; -import { getDataset, getDraftDataset } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; const apiId = "api.schema.validator"; export const eventValidation = async (req: Request, res: Response) => { @@ -25,12 +25,12 @@ export const eventValidation = async (req: Request, res: Response) => { } if (isLive) { - dataset = await getDataset(datasetId, true); + dataset = await datasetService.getDataset(datasetId, undefined, true); schema = _.get(dataset, "data_schema") } if (!isLive) { - dataset = await getDraftDataset(datasetId); + dataset = await datasetService.getDraftDataset(datasetId); schema = _.get(dataset, "data_schema") } diff --git a/api-service/src/v2/controllers/EventValidation/RequestValidationSchema.json b/api-service/src/controllers/EventValidation/RequestValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/EventValidation/RequestValidationSchema.json rename to api-service/src/controllers/EventValidation/RequestValidationSchema.json diff --git a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts new file mode 100644 index 00000000..a37e5d13 --- /dev/null +++ b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -0,0 +1,137 @@ +import httpStatus from "http-status"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { Request, Response } from "express"; +import { DataSetConfig, DatasetSchemaResponse, DatasetSchemeRequest } from "../../types/SchemaModel"; +import { obsrvError } from "../../types/ObsrvError"; +import { schemaValidation } from "../../services/ValidationService"; +import DataSchemaValidation from "./RequestValidationSchema.json" +import _ from "lodash"; +import constants from "../../services/SchemaGenerateService/Constants.json"; +import { SchemaInference } from "../../services/SchemaGenerateService/DataSchemaService"; +import { SchemaArrayValidator } from "../../services/SchemaGenerateService/SchemaArrayValidator"; +import { SchemaHandler } from "../../services/SchemaGenerateService/SchemaHandler"; +import DataMappings from "../../services/SchemaGenerateService/SchemaMapping.json"; +import { SchemaCardinalityAnalyser } from "../../services/SchemaGenerateService/SchemaCardinalityAnalyser"; +import { SuggestionTemplate } from "../../services/SchemaGenerateService/SuggestionTemplate"; +import { ConfigSuggestor } from "../../services/SchemaGenerateService/ConfigSuggester"; + +let rollupInfo = {} + +const validateRequest = async (req: Request) => { + + const isRequestValid: Record = schemaValidation(req.body, DataSchemaValidation) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATA_SCHEMA_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } + +} + +const dataSchema = async (req: Request, res: Response) => { + + await validateRequest(req) + const request = req.body.request + const dataSchemaSpec = schemaGenerate(request.data, request.config) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataSchemaSpec }); + +} + +const schemaGenerate = (sample: Map[], config: Record): any => { + const { isBatch = false, extractionKey, dataset } = config; + const isJsonSchema = checkJsonSchema(_.head(sample) || new Map()); + const schemaInference = new SchemaInference(); + const schemaArrayValidator = new SchemaArrayValidator(); + if (isJsonSchema) { + const result = process(sample, dataset) + result.schema = removeNonIndexColumns(result.schema) + result.schema = removeFormats(result.schema) + return result + } else { + let schema = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + schema = schemaArrayValidator.validate(schema) + const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) + rollupInfo = schemaCardinalityAnalyser.analyse() + const result = process(schema, dataset) + result.schema = removeNonIndexColumns(result.schema) + result.schema = removeFormats(result.schema) + return result + } +} + +const process = (schemas: Map[], dataset: string): DatasetSchemaResponse => { + const schemaHandler = new SchemaHandler(); + const suggestionTemplate = new SuggestionTemplate(); + const configSuggestor = new ConfigSuggestor(dataset); + const mergedSchema = schemaHandler.merge(schemas) + const report = schemaHandler.process(schemas) + const resolvedSchema = resolveConflicts(mergedSchema, report) + const suggestionTemplates = suggestionTemplate.createSuggestionTemplate(report) + const schema = schemaHandler.update(resolvedSchema, suggestionTemplates, "suggestions") + const suggestedConfig: DataSetConfig = configSuggestor.suggestConfig(report, rollupInfo) + const updatedSchema = schemaHandler.mapDataTypes(schema) + _.set(updatedSchema, "additionalProperties", true); + return { "schema": updatedSchema, "configurations": suggestedConfig, "dataMappings": DataMappings } +} + + +const checkJsonSchema = (sample: Map): boolean => { + const schemaProps = ["$id", "$schema", "$ref"] + return Object.keys(sample).some(key => schemaProps.includes(key)); +} + +const removeNonIndexColumns = (schema: any) => { + if (schema.properties) { + Object.entries(schema.properties).map(([, property]: any) => { + _.unset(schema, "required"); + removeNonIndexColumns(property) + }); + } else if (schema.items) { + removeNonIndexColumns(schema.items) + } + if (Array.isArray(schema.required) && schema.required.length === 0) { + _.unset(schema, "required"); + } + return schema +} + +const removeFormats = (schema: any) => { + if (schema.properties) { + Object.entries(schema.properties).map(([, property]: any) => { + // Removing format to avoid schema validation issues + const isDateTypeField = ["date-time", "date", "epoch"].includes((property as any).format); + if (isDateTypeField && _.get(property, "data_type") === "string") { + _.set(property, "data_type", _.get(property, "format")); + } else if (isDateTypeField && _.get(property, "data_type") === "integer") { + _.set(property, "data_type", "epoch"); + } + _.unset(property, "format"); + removeFormats(property) + }); + } else if (schema.items) { + _.unset(schema.items, "format"); + removeFormats(schema.items) + } + return schema +} + +const resolveConflicts = (schema: any, updateProps: any): any => { + const schemaHandler = new SchemaHandler(); + updateProps.map((value: any) => { + if (!_.isEmpty(value.schema) || !_.isEmpty(value.required)) { + switch (value.schema.type || value.required.type) { + case constants.SCHEMA_RESOLUTION_TYPE.DATA_TYPE: + return schemaHandler.update(schema, value, "datatype") + case constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD: + return schemaHandler.update(schema, value, "setNulltype") + case constants.SCHEMA_RESOLUTION_TYPE.OPTIONAL_TYPE: + return schemaHandler.update(schema, value, "required") + default: + console.warn("Unsupported Conflict Type") + break; + } + } else { console.info(`Conflicts not found ${JSON.stringify(value)}`) } + }) + return schema +} + + +export default dataSchema; \ No newline at end of file diff --git a/api-service/src/controllers/GenerateDataSchema/RequestValidationSchema.json b/api-service/src/controllers/GenerateDataSchema/RequestValidationSchema.json new file mode 100644 index 00000000..acffa0ec --- /dev/null +++ b/api-service/src/controllers/GenerateDataSchema/RequestValidationSchema.json @@ -0,0 +1,56 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.dataschema"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object" + }, + "minItems": 1 + }, + "config": { + "type": "object", + "properties": { + "dataset": { + "type": "string" + }, + "isBatch": { + "type": "boolean" + }, + "extractionKey": { + "type": "string" + } + }, + "required": ["dataset"] + } + }, + "required": ["data", "config"], + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts similarity index 100% rename from api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts rename to api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json rename to api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json diff --git a/api-service/src/v2/controllers/ListQueryTemplates/ListTemplateValidationSchema.json b/api-service/src/controllers/ListQueryTemplates/ListTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/ListQueryTemplates/ListTemplateValidationSchema.json rename to api-service/src/controllers/ListQueryTemplates/ListTemplateValidationSchema.json diff --git a/api-service/src/v2/controllers/ListQueryTemplates/ListTemplatesController.ts b/api-service/src/controllers/ListQueryTemplates/ListTemplatesController.ts similarity index 100% rename from api-service/src/v2/controllers/ListQueryTemplates/ListTemplatesController.ts rename to api-service/src/controllers/ListQueryTemplates/ListTemplatesController.ts diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts new file mode 100644 index 00000000..f6db4fd4 --- /dev/null +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -0,0 +1,122 @@ +import { Request, Response, NextFunction } from "express"; +import { Notification } from "../../models/Notification"; +import httpStatus from "http-status"; +import createError from "http-errors"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { publishNotificationChannel, testNotificationChannel, updateNotificationChannel } from "../../services/managers"; +import _ from "lodash"; +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; + +const createHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const payload = request.body; + const notificationBody = await Notification.create(payload); + updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const updateHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const updatedPayload = request.body; + const notificationPayloadModel = await Notification.findOne({ where: { id } }); + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + if (_.get(notificationPayload, "status") === "live") { + await updateNotificationChannel(notificationPayload); + } + await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const listHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { limit, filters, offset } = request.body?.request || {}; + const notifications = await Notification.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); + const count = _.get(notifications, "length"); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { notifications, ...(count && { count }) } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const fetchHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const notificationPayloadModel = await Notification.findOne({ where: { id } }); + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayloadModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: notificationPayloadModel?.toJSON() }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const retireHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const notificationPayloadModel = await Notification.findOne({ where: { id } }) + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + await updateNotificationChannel(notificationPayload); + await Notification.update({ status: "retired" }, { where: { id } }) + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const publishHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const notificationPayloadModel = await Notification.findOne({ where: { id } }) + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + await publishNotificationChannel(notificationPayload); + Notification.update({ status: "live" }, { where: { id } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const testNotifationChannelHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { message = "Hello Obsrv", payload = {} } = request.body; + const { id } = payload; + if (id) { + const notificationPayloadModel = await Notification.findOne({ where: { id } }) + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + await testNotificationChannel({ ...notificationPayload, message }); + } + else { + await testNotificationChannel({ ...payload, message }) + } + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "Notification Sent" } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +export default { listHandler, createHandler, publishHandler, updateHandler, retireHandler, fetchHandler, testNotifationChannelHandler } \ No newline at end of file diff --git a/api-service/src/v2/controllers/QueryTemplate/QueryTemplateController.ts b/api-service/src/controllers/QueryTemplate/QueryTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/QueryTemplate/QueryTemplateController.ts rename to api-service/src/controllers/QueryTemplate/QueryTemplateController.ts diff --git a/api-service/src/v2/controllers/QueryTemplate/QueryTemplateHelpers.ts b/api-service/src/controllers/QueryTemplate/QueryTemplateHelpers.ts similarity index 100% rename from api-service/src/v2/controllers/QueryTemplate/QueryTemplateHelpers.ts rename to api-service/src/controllers/QueryTemplate/QueryTemplateHelpers.ts diff --git a/api-service/src/v2/controllers/QueryTemplate/QueryTemplateValidationSchema.json b/api-service/src/controllers/QueryTemplate/QueryTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/QueryTemplate/QueryTemplateValidationSchema.json rename to api-service/src/controllers/QueryTemplate/QueryTemplateValidationSchema.json diff --git a/api-service/src/v2/controllers/QueryWrapper/SqlQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts similarity index 89% rename from api-service/src/v2/controllers/QueryWrapper/SqlQueryWrapper.ts rename to api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts index e1c0d561..dc1f0863 100644 --- a/api-service/src/v2/controllers/QueryWrapper/SqlQueryWrapper.ts +++ b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts @@ -1,12 +1,10 @@ -import axios from "axios"; import { Request, Response } from "express"; import _ from "lodash"; import { config } from "../../configs/Config"; import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; - -export const druidHttpService = axios.create({ baseURL: `${config.query_api.druid.host}:${config.query_api.druid.port}`, headers: { "Content-Type": "application/json" } }); +import { druidHttpService } from "../../connections/druidConnection"; const apiId = "api.obsrv.data.sql-query"; const errorCode = "SQL_QUERY_FAILURE" diff --git a/api-service/src/v2/controllers/ReadQueryTemplate/ReadTemplateController.ts b/api-service/src/controllers/ReadQueryTemplate/ReadTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/ReadQueryTemplate/ReadTemplateController.ts rename to api-service/src/controllers/ReadQueryTemplate/ReadTemplateController.ts diff --git a/api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateController.ts rename to api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts diff --git a/api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json rename to api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json diff --git a/api-service/src/exceptions/SchemaGenerationException.ts b/api-service/src/exceptions/SchemaGenerationException.ts new file mode 100644 index 00000000..fc545e81 --- /dev/null +++ b/api-service/src/exceptions/SchemaGenerationException.ts @@ -0,0 +1,8 @@ +export class SchemaGenerationException extends Error { + statusCode: number; + constructor(message: string, code: number) { + super(message); + this.name = "SchemaGenerationException"; + this.statusCode = code; + } +} \ No newline at end of file diff --git a/api-service/src/v2/helpers/ErrorResponseHandler.ts b/api-service/src/helpers/ErrorResponseHandler.ts similarity index 100% rename from api-service/src/v2/helpers/ErrorResponseHandler.ts rename to api-service/src/helpers/ErrorResponseHandler.ts diff --git a/api-service/src/v2/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts similarity index 64% rename from api-service/src/v2/helpers/ResponseHandler.ts rename to api-service/src/helpers/ResponseHandler.ts index c8d7e8b1..99aee816 100644 --- a/api-service/src/v2/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -1,9 +1,10 @@ import { NextFunction, Request, Response } from "express"; import httpStatus from "http-status"; import { IResponse, Result } from "../types/DatasetModels"; -import { onFailure, onSuccess } from "../metrics/prometheus/helpers"; +import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/helpers"; import moment from "moment"; import _ from "lodash"; +import { ObsrvError } from "../types/ObsrvError"; const ResponseHandler = { successResponse: (req: Request, res: Response, result: Result) => { @@ -18,7 +19,7 @@ const ResponseHandler = { next({ statusCode: httpStatus.NOT_FOUND, message: httpStatus["404"], errCode: httpStatus["404_NAME"] }); }, - refactorResponse: ({ id = "api", ver = "v1", params = { status: "SUCCESS" }, responseCode = httpStatus["200_NAME"], result = {}, msgid = "", resmsgid = "" }): IResponse => { + refactorResponse: ({ id = "api", ver = "v2", params = { status: "SUCCESS" }, responseCode = httpStatus["200_NAME"], result = {}, msgid = "", resmsgid = "" }): IResponse => { const paramsObj = { ...params, ...(!_.isEmpty(msgid) && { msgid }), resmsgid } return { id, ver, ts: moment().format(), params: paramsObj, responseCode, result } }, @@ -34,6 +35,16 @@ const ResponseHandler = { entity && onFailure(req, res) }, + obsrvErrorResponse: (error: ObsrvError, req: Request, res: Response) => { + const { statusCode, message, errCode, code = "INTERNAL_SERVER_ERROR", data } = error; + const { id, entity, body } = req as any; + const msgid = _.get(body, ["params", "msgid"]) + const resmsgid = _.get(res, "resmsgid") + const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) + res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); + entity && onObsrvFailure(req,res,error) + }, + setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { (req as any).id = id; next(); @@ -44,6 +55,11 @@ const ResponseHandler = { entity && onSuccess(req, res) res.status(result.status).send(result.data); }, + + goneResponse: (req: Request, res: Response) => { + const { id } = req as any; + res.status(httpStatus.GONE).json({ id: id, ver: "v1", ts: Date.now(), params: { status: "FAILED", errmsg: "v1 APIs have been replace by /v2 APIs. Please refer to this link for more information" }, responseCode: httpStatus["410_NAME"] }) + } } export { ResponseHandler }; diff --git a/api-service/src/logger/index.ts b/api-service/src/logger/index.ts new file mode 100644 index 00000000..d4d59f50 --- /dev/null +++ b/api-service/src/logger/index.ts @@ -0,0 +1,6 @@ +import log4js from "log4js"; + +const logger = log4js.getLogger(); +logger.level = "debug"; + +export default logger; \ No newline at end of file diff --git a/api-service/src/v2/metrics/prometheus/entities.ts b/api-service/src/metrics/prometheus/entities.ts similarity index 100% rename from api-service/src/v2/metrics/prometheus/entities.ts rename to api-service/src/metrics/prometheus/entities.ts diff --git a/api-service/src/v2/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts similarity index 54% rename from api-service/src/v2/metrics/prometheus/helpers.ts rename to api-service/src/metrics/prometheus/helpers.ts index 0a443f45..05051461 100644 --- a/api-service/src/v2/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -2,6 +2,7 @@ import { NextFunction, Response } from "express"; import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; +import { ObsrvError } from "../../types/ObsrvError"; export const onRequest = ({ entity = Entity.Management }: any) => (req: any, res: Response, next: NextFunction) => { const startTime = Date.now(); @@ -19,7 +20,7 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -28,18 +29,37 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + duration && setQueryResponseTime({ duration, labels }) + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + +export const onGone = (req: any, res: Response) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + const { statusCode = 410 } = res + const labels = { ...metricLabels, status: statusCode } + duration && setQueryResponseTime({ duration, labels }) + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + +export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + metricLabels.dataset_id = error.datasetId + const { statusCode = 404 } = res + const labels = { ...metricLabels, status: statusCode } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } const getMetricLabels = (req: any, res: Response) => { - const { id, entity, url, startTime } = req; + const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, "dataset_id") || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null const duration = getDuration(startTime); - const metricLabels = { entity, id, endpoint: url, dataset_id, status: statusCode, request_size, response_size } + const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } } diff --git a/api-service/src/v2/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts similarity index 95% rename from api-service/src/v2/metrics/prometheus/index.ts rename to api-service/src/metrics/prometheus/index.ts index da143f9f..173527ad 100644 --- a/api-service/src/v2/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -31,5 +31,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls }; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls}; diff --git a/api-service/src/v2/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts similarity index 100% rename from api-service/src/v2/metrics/prometheus/metrics.ts rename to api-service/src/metrics/prometheus/metrics.ts diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts new file mode 100644 index 00000000..d36babf0 --- /dev/null +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -0,0 +1,179 @@ +import { Request, Response, NextFunction } from "express"; +import jwt from "jsonwebtoken"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import { config } from "../configs/Config"; +import _ from "lodash"; + +enum roles { + Admin = "admin", + DatasetManager = "dataset_manager", + Viewer = "viewer", + DatasetCreator = "dataset_creator", + Ingestor = "ingestor", +} + +enum permissions { + DatasetCreate = "api.datasets.create", + DatasetUpdate = "api.datasets.update", + DatasetRead = "api.datasets.read", + DatasetList = "api.datasets.list", + DataIngest = "api.data.in", + DataOut = "api.data.out", + DataExhaust = "api.data.exhaust", + QueryTemplateCreate = "api.query.template.create", + QueryTemplateRead = "api.query.template.read", + QueryTemplateDelete = "api.query.template.delete", + QueryTemplateList = "api.query.template.list", + QueryTemplateUpdate = "api.query.template.update", + QueryTemplate = "api.query.template.query", + SchemaValidator = "api.schema.validator", + GenerateSignedUrl = "api.files.generate-url", + DatasetStatusTransition = "api.datasets.status-transition", + DatasetHealth = "api.dataset.health", + DatasetReset = "api.dataset.reset", + DatasetSchemaGenerator = "api.datasets.dataschema", + DatasetExport = "api.datasets.export", + DatasetCopy = "api.datasets.copy", + ConnectorList = "api.connectors.list", + ConnectorRead = "api.connectors.read", + DatasetImport = "api.datasets.import", + SQLQuery = "api.obsrv.data.sql-query", +} + +interface AccessControl { + [key: string]: string[]; +} + +const accessControl: AccessControl = { + [roles.Ingestor]: [permissions.DataIngest], + [roles.Viewer]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + ], + [roles.DatasetCreator]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + ], + [roles.DatasetManager]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition, + ], + [roles.Admin]: [ + permissions.DatasetCreate, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition, + ], +}; + +export default { + name: "rbac:middleware", + handler: () => (req: Request, res: Response, next: NextFunction) => { + try { + if (_.lowerCase(config.is_RBAC_enabled) === "false") { + next(); + } else { + const public_key = config.user_token_public_key; + const token = req.get("x-user-token"); + if (!token) { + return ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "No token provided", + }, + req, + res + ); + } + jwt.verify(token as string, public_key, (err, decoded) => { + if (err) { + return ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "Token verification failed", + }, + req, + res + ); + } + if (decoded && _.isObject(decoded)) { + const action = (req as any).id; + const hasAccess = decoded?.roles?.some( + (role: string) => + accessControl[role] && accessControl[role].includes(action) + ); + if (!hasAccess) { + return ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "Access denied for the user", + }, + req, + res + ); + } + next(); + } + }); + } + } catch (error) { + next(error); + } + }, +}; diff --git a/api-service/src/middlewares/errors.ts b/api-service/src/middlewares/errors.ts new file mode 100644 index 00000000..ce3200ab --- /dev/null +++ b/api-service/src/middlewares/errors.ts @@ -0,0 +1,19 @@ +import { NextFunction, Request, Response } from "express"; +import logger from "../logger"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import _ from "lodash"; +import { ObsrvError } from "../types/ObsrvError"; + +export const errorHandler = (err: Error, req: Request, res: Response, _next: NextFunction) => { + + logger.error({ path: req.url, req: req.body , ...err }) + const errorMessage = {name: err.name, message: err.message}; + ResponseHandler.errorResponse(errorMessage, req, res); +}; + + +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, _next: NextFunction) => { + + logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) + ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); +}; \ No newline at end of file diff --git a/api-service/src/v2/middlewares/setDataToRequestObject.ts b/api-service/src/middlewares/setDataToRequestObject.ts similarity index 100% rename from api-service/src/v2/middlewares/setDataToRequestObject.ts rename to api-service/src/middlewares/setDataToRequestObject.ts diff --git a/api-service/src/models/Alert.ts b/api-service/src/models/Alert.ts new file mode 100644 index 00000000..e52952b8 --- /dev/null +++ b/api-service/src/models/Alert.ts @@ -0,0 +1,74 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 as uuidv4 } from "uuid"; + +export const Alert = sequelize.define("alerts", { + id: { + type: DataTypes.STRING, + defaultValue: () => uuidv4().toString(), + primaryKey: true + }, + manager: { + type: DataTypes.STRING, + }, + name: { + type: DataTypes.STRING, + unique: true, + }, + status: { + type: DataTypes.ENUM("draft", "live", "retired"), + defaultValue: "draft", + }, + description: { + type: DataTypes.STRING, + }, + expression: { + type: DataTypes.STRING, + }, + severity: { + type: DataTypes.ENUM("warning", "critical"), + defaultValue: "warning" + }, + category: { + type: DataTypes.STRING, + }, + annotations: { + type: DataTypes.JSON, + defaultValue: {} + }, + labels: { + type: DataTypes.JSON, + defaultValue: {} + }, + frequency: { + type: DataTypes.STRING, + defaultValue: "1m", + }, + interval: { + type: DataTypes.STRING, + defaultValue: "1m", + }, + metadata: { + type: DataTypes.JSON, + defaultValue: {}, + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + notification: { + type: DataTypes.JSON, + defaultValue: {}, + }, + context: { + type: DataTypes.JSON, + defaultValue: {}, + } +}, { + tableName: "alerts" +}); + diff --git a/api-service/src/models/ConnectorInstances.ts b/api-service/src/models/ConnectorInstances.ts new file mode 100644 index 00000000..14c832d7 --- /dev/null +++ b/api-service/src/models/ConnectorInstances.ts @@ -0,0 +1,53 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const ConnectorInstances = sequelize.define("connector_instances", { + id: { + type: DataTypes.STRING, + primaryKey: true + }, + dataset_id: { + type: DataTypes.STRING + }, + connector_id: { + type: DataTypes.STRING + }, + data_format: { + type: DataTypes.STRING, + defaultValue: "json" + }, + connector_config: { + type: DataTypes.STRING + }, + operations_config: { + type: DataTypes.JSON, + defaultValue: {} + }, + status: { + type: DataTypes.ENUM("Publishing", "Live", "Retired") + }, + connector_state: { + type: DataTypes.JSON, + defaultValue: {} + }, + connector_stats: { + type: DataTypes.JSON, + defaultValue: {} + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + published_date: { + type: DataTypes.NUMBER + } +}, { + tableName: "connector_instances", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date" +}) \ No newline at end of file diff --git a/api-service/src/models/ConnectorRegistry.ts b/api-service/src/models/ConnectorRegistry.ts new file mode 100644 index 00000000..b31e5b27 --- /dev/null +++ b/api-service/src/models/ConnectorRegistry.ts @@ -0,0 +1,85 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const ConnectorRegistry = sequelize.define("connector_registry", { + id: { + type: DataTypes.STRING, + primaryKey: true + }, + connector_id: { + type: DataTypes.STRING, + allowNull: false, + unique: "connector_registry_unique" + }, + name: { + type: DataTypes.STRING, + allowNull: false + }, + type: { + type: DataTypes.STRING, + allowNull: false + }, + category: { + type: DataTypes.STRING, + allowNull: false + }, + version: { + type: DataTypes.STRING, + allowNull: false, + unique: "connector_registry_unique" + }, + description: { + type: DataTypes.STRING + }, + technology: { + type: DataTypes.STRING, + allowNull: false + }, + runtime: { + type: DataTypes.STRING, + allowNull: false + }, + licence: { + type: DataTypes.STRING, + allowNull: false + }, + owner: { + type: DataTypes.STRING, + allowNull: false + }, + iconURL: { + type: DataTypes.STRING + }, + status: { + type: DataTypes.ENUM("Draft", "InValidation", "Live", "Retired"), + defaultValue: "Draft", + }, + ui_spec: { + type: DataTypes.JSON, + defaultValue: {} + }, + source_url: { + type: DataTypes.STRING, + allowNull: false + }, + source: { + type: DataTypes.STRING, + allowNull: false + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + live_date: { + type: DataTypes.DATE + } +}, { + tableName: "connector_registry", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date" +}) \ No newline at end of file diff --git a/api-service/src/v2/models/Dataset.ts b/api-service/src/models/Dataset.ts similarity index 78% rename from api-service/src/v2/models/Dataset.ts rename to api-service/src/models/Dataset.ts index aebcc0fb..fa02cb10 100644 --- a/api-service/src/v2/models/Dataset.ts +++ b/api-service/src/models/Dataset.ts @@ -48,7 +48,7 @@ export const Dataset = sequelize.define("datasets", { defaultValue: {} }, status: { - type: DataTypes.ENUM("Draft", "Live", "Retired", "Publish"), + type: DataTypes.ENUM("Draft", "Publish", "Live", "Retired", "Archiving", "Archived"), defaultValue: "Draft", }, created_by: { @@ -61,10 +61,24 @@ export const Dataset = sequelize.define("datasets", { }, data_version: { type: DataTypes.NUMBER + }, + api_version: { + type: DataTypes.STRING, + }, + version: { + type: DataTypes.NUMBER + }, + sample_data: { + type: DataTypes.JSON, + defaultValue: {} + }, + entry_topic: { + type: DataTypes.STRING, + allowNull: false } }, { tableName: "datasets", timestamps: true, createdAt: "created_date", - updatedAt: "updated_date", + updatedAt: "updated_date" }) \ No newline at end of file diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/models/DatasetDraft.ts similarity index 85% rename from api-service/src/v2/models/DatasetDraft.ts rename to api-service/src/models/DatasetDraft.ts index 832989a1..30f56766 100644 --- a/api-service/src/v2/models/DatasetDraft.ts +++ b/api-service/src/models/DatasetDraft.ts @@ -55,6 +55,16 @@ export const DatasetDraft = sequelize.define("datasets_draft", { allowNull: true, defaultValue: {} }, + transformations_config: { + type: DataTypes.JSON, + allowNull: true, + defaultValue: {} + }, + connectors_config: { + type: DataTypes.JSON, + allowNull: true, + defaultValue: {} + }, tags: { type: DataTypes.ARRAY(DataTypes.TEXT), allowNull: true, @@ -80,15 +90,19 @@ export const DatasetDraft = sequelize.define("datasets_draft", { allowNull: false, defaultValue: "SYSTEM", }, - published_date: { + version_key: { + type: DataTypes.STRING, + allowNull: false + }, + api_version: { type: DataTypes.STRING, - allowNull: true + defaultValue: "v2" }, - client_state: { + sample_data: { type: DataTypes.JSON, defaultValue: {} }, - version_key: { + entry_topic: { type: DataTypes.STRING, allowNull: false } diff --git a/api-service/src/v2/models/DatasetSourceConfig.ts b/api-service/src/models/DatasetSourceConfig.ts similarity index 100% rename from api-service/src/v2/models/DatasetSourceConfig.ts rename to api-service/src/models/DatasetSourceConfig.ts diff --git a/api-service/src/v2/models/DatasetSourceConfigDraft.ts b/api-service/src/models/DatasetSourceConfigDraft.ts similarity index 90% rename from api-service/src/v2/models/DatasetSourceConfigDraft.ts rename to api-service/src/models/DatasetSourceConfigDraft.ts index c7a8e889..df84c1a6 100644 --- a/api-service/src/v2/models/DatasetSourceConfigDraft.ts +++ b/api-service/src/models/DatasetSourceConfigDraft.ts @@ -1,5 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; +import { DatasetStatus } from "../types/DatasetModels"; export const DatasetSourceConfigDraft = sequelize.define("dataset_source_config_draft", { id: { @@ -18,6 +19,7 @@ export const DatasetSourceConfigDraft = sequelize.define("dataset_source_config_ }, status: { type: DataTypes.STRING, + defaultValue: DatasetStatus.Draft }, connector_stats: { type: DataTypes.JSON, @@ -30,9 +32,6 @@ export const DatasetSourceConfigDraft = sequelize.define("dataset_source_config_ updated_by: { type: DataTypes.STRING, defaultValue: "SYSTEM", - }, - published_date: { - type: DataTypes.TIME } }, { tableName: "dataset_source_config_draft", diff --git a/api-service/src/v2/models/Datasource.ts b/api-service/src/models/Datasource.ts similarity index 92% rename from api-service/src/v2/models/Datasource.ts rename to api-service/src/models/Datasource.ts index 9896b230..6b79890b 100644 --- a/api-service/src/v2/models/Datasource.ts +++ b/api-service/src/models/Datasource.ts @@ -39,6 +39,11 @@ export const Datasource = sequelize.define("datasources", { type: DataTypes.ENUM("Draft", "Live", "Retired", "Publish"), defaultValue: "Draft", }, + type: { + type: DataTypes.STRING, + allowNull: false, + defaultValue: "druid", + }, created_by: { type: DataTypes.STRING, defaultValue: "SYSTEM", diff --git a/api-service/src/v2/models/DatasourceDraft.ts b/api-service/src/models/DatasourceDraft.ts similarity index 93% rename from api-service/src/v2/models/DatasourceDraft.ts rename to api-service/src/models/DatasourceDraft.ts index 272f5a9d..7b211358 100644 --- a/api-service/src/v2/models/DatasourceDraft.ts +++ b/api-service/src/models/DatasourceDraft.ts @@ -39,6 +39,11 @@ export const DatasourceDraft = sequelize.define("datasources_draft", { type: DataTypes.ENUM("Draft", "Live", "Retired", "Publish"), defaultValue: "Draft", }, + type: { + type: DataTypes.STRING, + allowNull: false, + defaultValue: "druid", + }, created_by: { type: DataTypes.STRING, defaultValue: "SYSTEM", @@ -47,9 +52,6 @@ export const DatasourceDraft = sequelize.define("datasources_draft", { type: DataTypes.STRING, defaultValue: "SYSTEM", }, - published_date: { - type: DataTypes.TIME - }, metadata: { type: DataTypes.JSON, defaultValue: { "aggregated": false, "granularity": "day" } diff --git a/api-service/src/models/Metric.ts b/api-service/src/models/Metric.ts new file mode 100644 index 00000000..ecc4cb27 --- /dev/null +++ b/api-service/src/models/Metric.ts @@ -0,0 +1,30 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 as uuidv4 } from "uuid"; + +export const Metrics = sequelize.define("metrics", { + id: { + type: DataTypes.STRING, + defaultValue: () => uuidv4().toString(), + primaryKey: true + }, + alias: { + type: DataTypes.STRING, + unique: true, + }, + component: { + type: DataTypes.STRING, + }, + subComponent: { + type: DataTypes.STRING, + }, + metric: { + type: DataTypes.STRING, + }, + context: { + type: DataTypes.JSON, + defaultValue: {} + } +}, { + tableName: "metrics" +}); \ No newline at end of file diff --git a/api-service/src/models/Notification.ts b/api-service/src/models/Notification.ts new file mode 100644 index 00000000..6203032c --- /dev/null +++ b/api-service/src/models/Notification.ts @@ -0,0 +1,46 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 } from "uuid"; + +export const Notification = sequelize.define("notificationchannel", { + id: { + type: DataTypes.STRING, + defaultValue: () => v4().toString(), + primaryKey: true + }, + manager: { + type: DataTypes.STRING + }, + name: { + type: DataTypes.STRING, + unique: true, + }, + status: { + type: DataTypes.ENUM("draft", "live", "retired"), + defaultValue: "draft", + }, + type: { + type: DataTypes.ENUM("Slack"), + defaultValue: "Slack", + }, + config: { + type: DataTypes.JSON, + defaultValue: {} + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + context: { + type: DataTypes.JSON, + defaultValue: {}, + } +}, { + tableName: "notificationChannel" +}); + + diff --git a/api-service/src/v2/models/QueryTemplate.ts b/api-service/src/models/QueryTemplate.ts similarity index 100% rename from api-service/src/v2/models/QueryTemplate.ts rename to api-service/src/models/QueryTemplate.ts diff --git a/api-service/src/models/Silence.ts b/api-service/src/models/Silence.ts new file mode 100644 index 00000000..d78d3f89 --- /dev/null +++ b/api-service/src/models/Silence.ts @@ -0,0 +1,38 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 as uuid } from "uuid"; + +export const Silence = sequelize.define("silences", { + id: { + type: DataTypes.STRING, + defaultValue: () => uuid().toString(), + primaryKey: true + }, + manager: { + type: DataTypes.STRING, + }, + start_time: { + type: DataTypes.DATE, + }, + end_time: { + type: DataTypes.DATE, + }, + alert_id: { + type: DataTypes.STRING, + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "ADMIN", + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "ADMIN", + }, + context: { + type: DataTypes.JSON, + defaultValue: {}, + } +}, { + tableName: "silences", + timestamps: true, +}); \ No newline at end of file diff --git a/api-service/src/v2/models/Transformation.ts b/api-service/src/models/Transformation.ts similarity index 100% rename from api-service/src/v2/models/Transformation.ts rename to api-service/src/models/Transformation.ts diff --git a/api-service/src/v2/models/TransformationDraft.ts b/api-service/src/models/TransformationDraft.ts similarity index 93% rename from api-service/src/v2/models/TransformationDraft.ts rename to api-service/src/models/TransformationDraft.ts index 601d5c59..55cb0264 100644 --- a/api-service/src/v2/models/TransformationDraft.ts +++ b/api-service/src/models/TransformationDraft.ts @@ -41,10 +41,6 @@ export const DatasetTransformationsDraft = sequelize.define("dataset_transformat type: DataTypes.TEXT, allowNull: false, defaultValue: "SYSTEM", - }, - published_date: { - type: DataTypes.DATE, - defaultValue: null } }, { timestamps: true, diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts new file mode 100644 index 00000000..01c843dd --- /dev/null +++ b/api-service/src/routes/AlertsRouter.ts @@ -0,0 +1,41 @@ +import express from "express"; +import notificationHandler from "../controllers/NotificationChannel/Notification"; +import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; +import customAlertHandler from "../controllers/Alerts/Alerts"; +import metricAliasHandler from "../controllers/Alerts/Metric"; +import silenceHandler from "../controllers/Alerts/Silence"; + +export const alertsRouter = express.Router(); + +// Notifications + +alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), notificationHandler.listHandler); +alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), notificationHandler.createHandler); +alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), notificationHandler.publishHandler); +alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), notificationHandler.testNotifationChannelHandler); +alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), notificationHandler.updateHandler); +alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), notificationHandler.retireHandler); +alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), notificationHandler.fetchHandler); + +// alerts +alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), customAlertHandler.createAlertHandler); +alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), customAlertHandler.publishAlertHandler); +alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), customAlertHandler.searchAlertHandler); +alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), customAlertHandler.alertDetailsHandler); +alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteAlertHandler); +alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteSystemAlertsHandler); +alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), customAlertHandler.updateAlertHandler); + +// metrics +alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), metricAliasHandler.createMetricHandler); +alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), metricAliasHandler.listMetricsHandler); +alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"),metricAliasHandler.updateMetricHandler); +alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"),metricAliasHandler.deleteMetricHandler); +alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), metricAliasHandler.deleteMultipleMetricHandler); + +// silence +alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"),silenceHandler.createHandler); +alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"),silenceHandler.listHandler); +alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"),silenceHandler.fetchHandler); +alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"),silenceHandler.updateHandler); +alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"),silenceHandler.deleteHandler); \ No newline at end of file diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts new file mode 100644 index 00000000..ca1aec91 --- /dev/null +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -0,0 +1,23 @@ +import express from "express"; +import { Entity } from "../types/MetricModel"; +import { wrapperService } from "../services/WrapperService"; +import { onRequest } from "../metrics/prometheus/helpers"; +import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; +import { healthService } from "../services/HealthService"; +import { ResponseHandler } from "../helpers/ResponseHandler"; + +export const druidProxyRouter = express.Router(); + +// Send a 410 Gone response to all V1 API calls +druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) + +// Druid Proxy APIs for Metabase integration +druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) +druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) +druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) +druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file diff --git a/api-service/src/v2/routes/metricRouter.ts b/api-service/src/routes/MetricRouter.ts similarity index 56% rename from api-service/src/v2/routes/metricRouter.ts rename to api-service/src/routes/MetricRouter.ts index 92ce978c..3becdd79 100644 --- a/api-service/src/v2/routes/metricRouter.ts +++ b/api-service/src/routes/MetricRouter.ts @@ -1,6 +1,6 @@ import express from "express"; import { metricsScrapeHandler } from "../metrics/prometheus"; -export const router = express.Router(); +export const metricRouter = express.Router(); //Scrape metrics to prometheus -router.get("/metrics", metricsScrapeHandler) \ No newline at end of file +metricRouter.get("/metrics", metricsScrapeHandler) \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts new file mode 100644 index 00000000..40f1df02 --- /dev/null +++ b/api-service/src/routes/Router.ts @@ -0,0 +1,65 @@ +import express from "express"; +import dataIn from "../controllers/DataIngestion/DataIngestionController"; +import DatasetCreate from "../controllers/DatasetCreate/DatasetCreate"; +import dataOut from "../controllers/DataOut/DataOutController"; +import DatasetUpdate from "../controllers/DatasetUpdate/DatasetUpdate"; +import DatasetRead from "../controllers/DatasetRead/DatasetRead"; +import DatasetList from "../controllers/DatasetList/DatasetList" +import { dataExhaust } from "../controllers/DataExhaust/DataExhaustController"; +import { onRequest } from "../metrics/prometheus/helpers"; +import { Entity } from "../types/MetricModel"; +import { createQueryTemplate } from "../controllers/CreateQueryTemplate/CreateTemplateController"; +import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; +import { readQueryTemplate } from "../controllers/ReadQueryTemplate/ReadTemplateController"; +import { deleteQueryTemplate } from "../controllers/DeleteQueryTemplate/DeleteTemplateController"; +import { listQueryTemplates } from "../controllers/ListQueryTemplates/ListTemplatesController"; +import { queryTemplate } from "../controllers/QueryTemplate/QueryTemplateController"; +import { updateQueryTemplate } from "../controllers/UpdateQueryTemplate/UpdateTemplateController"; +import { eventValidation } from "../controllers/EventValidation/EventValidation"; +import GenerateSignedURL from "../controllers/GenerateSignedURL/GenerateSignedURL"; +import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; +import DatasetStatusTansition from "../controllers/DatasetStatusTransition/DatasetStatusTransition"; +import datasetHealth from "../controllers/DatasetHealth/DatasetHealth"; +import DataSchemaGenerator from "../controllers/GenerateDataSchema/GenerateDataSchema"; +import datasetReset from "../controllers/DatasetReset/DatasetReset"; +import DatasetExport from "../controllers/DatasetExport/DatasetExport"; +import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; +import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; +import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; +import DatasetImport from "../controllers/DatasetImport/DatasetImport"; +import { OperationType, telemetryAuditStart } from "../services/telemetry"; +import telemetryActions from "../telemetry/telemetryActions"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; +import checkRBAC from "../middlewares/RBAC_middleware"; + +export const router = express.Router(); + +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), checkRBAC.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), checkRBAC.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), checkRBAC.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), checkRBAC.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); + +//Wrapper Service +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) + diff --git a/api-service/src/services/CipherService.ts b/api-service/src/services/CipherService.ts new file mode 100644 index 00000000..5f85198d --- /dev/null +++ b/api-service/src/services/CipherService.ts @@ -0,0 +1,30 @@ +import crypto from "crypto"; +import { config } from "../configs/Config"; + +class CipherService { + public encrypt(data: string) { + const cipher = crypto.createCipheriv( + config.encryption_config.encryption_algorithm, + config.encryption_config.encryption_key, + "", + ) + const toEncrypt = Buffer.from(data, "utf8"); + let encryptedString = cipher.update(toEncrypt); + encryptedString = Buffer.concat([encryptedString, cipher.final()]) + return encryptedString.toString("base64"); + } + + public decrypt(data: string) { + const decipher = crypto.createDecipheriv( + config.encryption_config.encryption_algorithm, + config.encryption_config.encryption_key, + "", + ) + const encryptedText = Buffer.from(data, "base64"); + let decryptedString = decipher.update(encryptedText); + decryptedString = Buffer.concat([decryptedString, decipher.final()]) + return decryptedString.toString(); + } +} + +export const cipherService = new CipherService() \ No newline at end of file diff --git a/api-service/src/v2/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts similarity index 75% rename from api-service/src/v2/services/CloudServices/AWSStorageService.ts rename to api-service/src/services/CloudServices/AWSStorageService.ts index 44554329..d83b65b1 100644 --- a/api-service/src/v2/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -5,21 +5,27 @@ import { config as globalConfig } from "../../configs/Config"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner" import { getFileKey } from "../../utils/common" import { FilterDataByDateRange, ICloudService } from "./types"; -import { logger } from "@azure/storage-blob"; import { URLAccess } from "../../types/SampleURLModel"; +import logger from "../../logger"; export class AWSStorageService implements ICloudService { client: any; constructor(config: any) { if (_.get(config, "identity") && _.get(config, "credential") && _.get(config, "region")) { - const region = _.get(config, "region").toString(); - this.client = new S3Client({ region }); - } else { - const region = globalConfig.cloud_config.cloud_storage_region || "us-east-2"; - const s3Client = new S3Client({ - region, - }); - this.client = s3Client; + const region = _.get(config, "region") + const accessKeyId = _.get(config, "identity") + const secretAccessKey = _.get(config, "credential") + const endpoint = _.get(config, "endpoint") + const configuration: any = { region, credentials: { accessKeyId, secretAccessKey } } + if(endpoint) { + configuration.endpoint = endpoint; + } + try { + this.client = new S3Client(configuration); + } + catch (err) { + logger.error(err) + } } } @@ -39,12 +45,18 @@ export class AWSStorageService implements ICloudService { const AWSCommand = access === URLAccess.Read ? this.getAWSCommand : this.putAWSCommand const containerURLExpiry = urlExpiry ? urlExpiry : globalConfig.cloud_config.storage_url_expiry const signedURLs = filesList.map((fileNameWithPrefix: any) => { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const generateSignedUrl = async () => { - const command = AWSCommand(container, fileNameWithPrefix); - const fileName = fileNameWithPrefix.split("/").pop(); - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: containerURLExpiry }); - resolve({ [fileName]: presignedURL }); + try { + const command = AWSCommand(container, fileNameWithPrefix); + const fileName = fileNameWithPrefix.split("/").pop(); + const presignedURL = await getSignedUrl(this.client, command, { expiresIn: containerURLExpiry }); + resolve({ [fileName]: presignedURL }); + } + catch (err: any) { + logger.error({ error: err?.message }) + reject({ error: err?.message }); + } } generateSignedUrl(); }); @@ -55,7 +67,7 @@ export class AWSStorageService implements ICloudService { async getSignedUrls(container: any, filesList: any) { const signedUrlsPromises = this.generateSignedURLs(container, filesList) const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; const files: any[] = []; // Formatting response signedUrlsList.map(async (fileObject) => { diff --git a/api-service/src/v2/services/CloudServices/AzureStorageService.ts b/api-service/src/services/CloudServices/AzureStorageService.ts similarity index 99% rename from api-service/src/v2/services/CloudServices/AzureStorageService.ts rename to api-service/src/services/CloudServices/AzureStorageService.ts index c90af51e..b4472aac 100644 --- a/api-service/src/v2/services/CloudServices/AzureStorageService.ts +++ b/api-service/src/services/CloudServices/AzureStorageService.ts @@ -130,7 +130,7 @@ export class AzureStorageService implements ICloudService { const signedUrlsPromises = this.generateSignedURLs(container, filesList) const signedUrlsList = await Promise.all(signedUrlsPromises); const files: any[] = [] - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; // Formatting response signedUrlsList.map(async (fileObject) => { const fileDetails = _.keys(fileObject); diff --git a/api-service/src/v2/services/CloudServices/GCPStorageService.ts b/api-service/src/services/CloudServices/GCPStorageService.ts similarity index 98% rename from api-service/src/v2/services/CloudServices/GCPStorageService.ts rename to api-service/src/services/CloudServices/GCPStorageService.ts index 90fcaeb0..b07e3e4e 100644 --- a/api-service/src/v2/services/CloudServices/GCPStorageService.ts +++ b/api-service/src/services/CloudServices/GCPStorageService.ts @@ -74,7 +74,7 @@ export class GCPStorageService implements ICloudService { return generateSignedUrls() .then(signedUrlList => { - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; const files: any = []; const signedUrls = _.flattenDeep(_.map(signedUrlList, url => { const values = _.values(url) diff --git a/api-service/src/v2/services/CloudServices/index.ts b/api-service/src/services/CloudServices/index.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/index.ts rename to api-service/src/services/CloudServices/index.ts diff --git a/api-service/src/v2/services/CloudServices/types.ts b/api-service/src/services/CloudServices/types.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/types.ts rename to api-service/src/services/CloudServices/types.ts diff --git a/api-service/src/services/ConnectorService.ts b/api-service/src/services/ConnectorService.ts new file mode 100644 index 00000000..ddbb211b --- /dev/null +++ b/api-service/src/services/ConnectorService.ts @@ -0,0 +1,15 @@ +import { ConnectorRegistry } from "../models/ConnectorRegistry"; + +class ConnectorService { + + findConnectors = (where?: Record, attributes?: string[]): Promise => { + return ConnectorRegistry.findAll({ where, attributes, raw: true }); + } + + getConnector = (where?: Record, attributes?: string[]): Promise => { + return ConnectorRegistry.findOne({ where, attributes, raw: true }); + } + +} + +export const connectorService = new ConnectorService(); \ No newline at end of file diff --git a/api-service/src/services/DatasetHealthService.ts b/api-service/src/services/DatasetHealthService.ts new file mode 100644 index 00000000..5ea8b6df --- /dev/null +++ b/api-service/src/services/DatasetHealthService.ts @@ -0,0 +1,513 @@ +import axios from "axios"; +import { config } from "../configs/Config"; +import { logger } from "@project-sunbird/logger"; +import { health as postgresHealth } from "../connections/databaseConnection"; +import { DatasetType, DataSourceType, HealthStatus } from "../types/DatasetModels"; +import { createClient } from "redis"; +import { isHealthy as isKafkaHealthy } from "../connections/kafkaConnection"; +import { druidHttpService, executeNativeQuery } from "../connections/druidConnection"; +import _ from "lodash"; +import moment from "moment"; +import { SystemConfig } from "./SystemConfig"; +import { datasetService } from "./DatasetService"; +const dateFormat = "YYYY-MM-DDT00:00:00+05:30" + +const prometheusInstance = axios.create({ baseURL: config?.query_api?.prometheus?.url, headers: { "Content-Type": "application/json" } }) + +const prometheusQueries = { + validationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_validator_failed_count[1d]))", + dedupFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_dedup_failed_count[1d]))", + denormFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_DATASETID_denorm_failed[1d]))", + transformationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_DATASETID_transform_failed_count[1d]))", + queriesCount: "sum(sum_over_time(node_total_api_calls{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))", + avgQueryResponseTimeInSec: "avg(avg_over_time(node_query_response_time{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))/1000", + queriesFailedCount: "sum(sum_over_time(node_failed_api_calls{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))" +} + +export const getDatasetHealth = async (categories: any, dataset: any) => { + + const details = [] + if (categories.includes("infra")) { + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; + const { components, status } = await getInfraHealth(isMasterDataset) + details.push({ + "category": "infra", + "status": status, + "components": components + }) + } + if (categories.includes("processing")) { + const { components, status } = await getProcessingHealth(dataset) + details.push({ + "category": "processing", + "status": status, + "components": components + }) + } + + if (categories.includes("query")) { + const datasources = await datasetService.findDatasources({ dataset_id: dataset.id, type: DataSourceType.druid }, ["dataset_id", "datasource"]) + const { components, status } = await getQueryHealth(datasources, dataset) + details.push({ + "category": "query", + "status": status, + "components": components + }) + } + + const allStatus = _.includes(_.map(details, (detail) => detail?.status), HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy + return { + "status": allStatus, + "details": details + } +} +const getDatasetIdForMetrics = (datasetId: string) => { + datasetId = datasetId.replace(/-/g, "_") + .replace(/\./g, "_") + .replace(/\n/g, "") + .replace(/[\n\r]/g, "") + return datasetId; +} + +const queryMetrics = async (datasetId: string, query: string) => { + const queryWithDatasetId = query.replace("DATASETID", getDatasetIdForMetrics(datasetId)) + try { + const { data } = await prometheusInstance.get("/api/v1/query", { params: {query: queryWithDatasetId} }) + return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } + +} + +export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ components: any, status: string }> => { + const postgres = await getPostgresStatus() + const druid = await getDruidHealthStatus() + const flink = await getFlinkHealthStatus() + const kafka = await getKafkaHealthStatus() + let redis = HealthStatus.Healthy + const components = [ + { "type": "postgres", "status": postgres }, + { "type": "kafka", "status": kafka }, + { "type": "druid", "status": druid }, + { "type": "flink", "status": flink } + ] + if (isMasterDataset) { + redis = await getRedisHealthStatus() + components.push({ "type": "redis", "status": redis }) + } + const status = [postgres, redis, kafka, druid, flink].includes(HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy + return { components, status }; +} + +export const getProcessingHealth = async (dataset: any): Promise<{ components: any, status: string }> => { + const dataset_id = _.get(dataset, "dataset_id") + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; + const flink = await getFlinkHealthStatus() + const { count, health } = await getEventsProcessedToday(dataset_id, isMasterDataset) + const processingDefaultThreshold = await SystemConfig.getThresholds("processing") + // eslint-disable-next-line prefer-const + let { count: avgCount, health: avgHealth } = await getAvgProcessingSpeedInSec(dataset_id, isMasterDataset) + if (avgHealth == HealthStatus.Healthy) { + if (avgCount > processingDefaultThreshold?.avgProcessingSpeedInSec) { + avgHealth = HealthStatus.UnHealthy + } + } + const failure = await queryMetrics(dataset_id, prometheusQueries.validationFailure) + failure.health = getProcessingComponentHealth(failure, count, processingDefaultThreshold?.validationFailuresCount) + + const dedupFailure = await queryMetrics(dataset_id, prometheusQueries.dedupFailure) + dedupFailure.health = getProcessingComponentHealth(dedupFailure, count, processingDefaultThreshold?.dedupFailuresCount) + + const denormFailure = await queryMetrics(dataset_id, prometheusQueries.denormFailure) + denormFailure.health = getProcessingComponentHealth(denormFailure, count, processingDefaultThreshold?.denormFailureCount) + + const transformFailure = await queryMetrics(dataset_id, prometheusQueries.transformationFailure) + denormFailure.health = getProcessingComponentHealth(transformFailure, count, processingDefaultThreshold?.transformFailureCount) + + const components = [ + { + "type": "pipeline", + "status": flink + }, + { + "type": "eventsProcessedCount", + "count": count, + "status": health + }, + { + "type": "avgProcessingSpeedInSec", + "count": avgCount, + "status": avgHealth + }, + { + "type": "validationFailuresCount", + "count": failure?.count, + "status": failure?.health + }, + { + "type": "dedupFailuresCount", + "count": dedupFailure?.count, + "status": dedupFailure?.health + }, + { + "type": "denormFailureCount", + "count": denormFailure?.count, + "status": denormFailure?.health + }, + { + "type": "transformFailureCount", + "count": transformFailure?.count, + "status": transformFailure?.health + } + ] + const Healths = _.map(components, (component: any) => component?.status) + + const status = _.includes(Healths, HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy; + + return { components, status }; +} + +const getProcessingComponentHealth = (info: any, count: any, threshold: any) => { + let status = info.health + logger.debug({ info, count, threshold }) + if (info.health == HealthStatus.Healthy) { + if (info.count > 0 && count == 0) { + status = HealthStatus.UnHealthy + } else { + const percentage = (info.count / count) * 100 + if (percentage > threshold) { + status = HealthStatus.UnHealthy + } + } + } + return status; +} + +export const getQueryHealth = async (datasources: any, dataset: any): Promise<{ components: any, status: string }> => { + + const components: any = []; + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; + let status = HealthStatus.Healthy; + if (!isMasterDataset) { + if (!_.isEmpty(datasources)) { + const druidTasks = await getDruidIndexerStatus(datasources); + components.push( + { + "type": "indexer", + "status": _.get(druidTasks, "status"), + "value": _.get(druidTasks, "value") + } + ) + if (_.get(druidTasks, "status") == HealthStatus.UnHealthy) { + status = HealthStatus.UnHealthy + } + } else { + components.push({ + "type": "indexer", + "status": HealthStatus.UnHealthy, + "value": [] + }) + status = HealthStatus.UnHealthy + } + } + + + const queriesCount = await queryMetrics(dataset?.dataset_id, prometheusQueries.queriesCount) + const defaultThresholds = await SystemConfig.getThresholds("query") + + components.push({ + "type": "queriesCount", + "count": queriesCount.count, + "status": queriesCount.health + }) + + const avgQueryReponseTimeInSec = await queryMetrics(dataset?.dataset_id, prometheusQueries.avgQueryResponseTimeInSec) + if (avgQueryReponseTimeInSec.count > defaultThresholds?.avgQueryReponseTimeInSec) { + avgQueryReponseTimeInSec.health = HealthStatus.UnHealthy + } + components.push({ + "type": "avgQueryReponseTimeInSec", + "count": avgQueryReponseTimeInSec.count, + "status": avgQueryReponseTimeInSec.health + }) + + const queriesFailed = await queryMetrics(dataset?.dataset_id, prometheusQueries.queriesFailedCount) + if (queriesCount.count == 0 && queriesFailed.count > 0) { + queriesFailed.health = HealthStatus.UnHealthy + } else { + const percentage = (queriesFailed.count / queriesCount.count) * 100; + if (percentage > defaultThresholds?.queriesFailed) { + queriesFailed.health = HealthStatus.UnHealthy + } + } + if ([queriesFailed.health, avgQueryReponseTimeInSec.health].includes(HealthStatus.UnHealthy)) { + status = HealthStatus.UnHealthy + } + + components.push({ + "type": "queriesFailed", + "count": queriesFailed.count, + "status": queriesFailed.health + }) + return { components, status } +} + +const getDruidIndexerStatus = async (datasources: any,) => { + try { + const results = await Promise.all(_.map(datasources, (datasource) => getDruidDataourceStatus(datasource["datasource"]))) + const values: any = [] + let status = HealthStatus.Healthy + _.forEach(results, (result: any) => { + logger.debug({ result }) + const sourceStatus = _.get(result, "payload.state") == "RUNNING" ? HealthStatus.Healthy : HealthStatus.UnHealthy + logger.debug({ sourceStatus }) + values.push( + { + "type": "druid", + "datasource": _.get(result, "id"), + "status": sourceStatus, + } + ) + if (sourceStatus == HealthStatus.UnHealthy) { + status = HealthStatus.UnHealthy + } + }) + return { value: values, status } + } catch (error) { + logger.error(error) + return { value: [], status: HealthStatus.UnHealthy } + } +} + +const getDruidDataourceStatus = async (datasourceId: string) => { + logger.debug(datasourceId) + const { data } = await druidHttpService.get(`/druid/indexer/v1/supervisor/${datasourceId}/status`) + return data; +} + +const getPostgresStatus = async (): Promise => { + try { + await postgresHealth() + } catch (error) { + logger.error(error) + return HealthStatus.UnHealthy + } + return HealthStatus.Healthy +} + +const connectToRedis = async (url: string) => { + return new Promise((resolve, reject) => { + createClient({ + url + }) + .on("error", (err: any) => { + reject(err) + }) + .on("connect", () => { + resolve("connected") + }) + .connect(); + }) +} + +const getRedisHealthStatus = async () => { + try { + await Promise.all([connectToRedis(`redis://${config.redis_config.denorm_redis_host}:${config.redis_config.denorm_redis_port}`), + connectToRedis(`redis://${config.redis_config.dedup_redis_host}:${config.redis_config.dedup_redis_port}`)]); + return HealthStatus.Healthy; + } catch (error) { + logger.error(error) + } + return HealthStatus.UnHealthy; +} + +const getKafkaHealthStatus = async () => { + try { + const status = await isKafkaHealthy() + return status ? HealthStatus.Healthy : HealthStatus.UnHealthy + } catch (error) { + return HealthStatus.UnHealthy + } + +} + +export const getFlinkHealthStatus = async () => { + try { + const responses = await Promise.all( + [axios.get(config?.flink_job_configs?.masterdata_processor_job_manager_url as string + "/jobs"), + axios.get(config?.flink_job_configs?.pipeline_merged_job_manager_url as string + "/jobs")] + ) + const isHealthy = _.every(responses, (response: any) => { + const { data = {} } = response; + return _.get(data, "jobs[0].status") === "RUNNING" + }) + return isHealthy ? HealthStatus.Healthy : HealthStatus.UnHealthy; + } catch (error) { + logger.error("Unable to get flink status", error) + } + return HealthStatus.UnHealthy; +} + +const getDruidHealthStatus = async () => { + try { + const { data = false } = await druidHttpService.get("/status/health") + return data ? HealthStatus.Healthy : HealthStatus.UnHealthy + } catch (error) { + logger.error(error) + return HealthStatus.UnHealthy + } +} + +const getEventsProcessedToday = async (datasetId: string, isMasterDataset: boolean) => { + const startDate = moment().format(dateFormat); + const endDate = moment().add(1, "d").format(dateFormat); + const intervals = `${startDate}/${endDate}` + logger.debug({ datasetId, isMasterDataset }) + try { + const { data } = await executeNativeQuery({ + "queryType": "timeseries", + "dataSource": "system-events", + "intervals": intervals, + "granularity": { + "type": "all", + "timeZone": "Asia/Kolkata" + }, + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "ctx_module", + "value": "processing" + }, + { + "type": "selector", + "dimension": "ctx_dataset", + "value": datasetId + }, + { + "type": "selector", + "dimension": "ctx_pdata_id", + "value": isMasterDataset ? "MasterDataProcessorJob" : "DruidRouterJob" + }, + { + "type": "selector", + "dimension": "error_code", + "value": null + } + ] + }, + "aggregations": [ + { + "type": "longSum", + "name": "count", + "fieldName": "count" + } + ] + }) + return { health: HealthStatus.Healthy, count: _.get(data, "[0].result.count", 0) || 0 } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } +} + +const getAvgProcessingSpeedInSec = async (datasetId: string, isMasterDataset: boolean) => { + const startDate = moment().format(dateFormat); + const endDate = moment().add(1, "d").format(dateFormat); + const intervals = `${startDate}/${endDate}` + logger.debug({ datasetId, isMasterDataset }) + try { + const { data } = await executeNativeQuery({ + "queryType": "groupBy", + "dataSource": "system-events", + "intervals": intervals, + "granularity": { + "type": "all", + "timeZone": "Asia/Kolkata" + }, + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "ctx_module", + "value": "processing" + }, + { + "type": "selector", + "dimension": "ctx_dataset", + "value": datasetId + }, + { + "type": "selector", + "dimension": "ctx_pdata_id", + "value": isMasterDataset ? "MasterDataProcessorJob" : "DruidRouterJob" + }, + { + "type": "selector", + "dimension": "error_code", + "value": null + } + ] + }, + "aggregations": [ + { + "type": "longSum", + "name": "processing_time", + "fieldName": "total_processing_time" + }, + { + "type": "longSum", + "name": "count", + "fieldName": "count" + } + ], + "postAggregations": [ + { + "type": "expression", + "name": "average_processing_time", + "expression": "case_searched((count > 0),(processing_time/count),0", + } + ] + }) + logger.debug({ average_processing_time: JSON.stringify(data) }) + const count = _.get(data, "[0].event.average_processing_time", 0) || 0 + return { health: HealthStatus.Healthy, count: count / 1000 } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } +} + +export const getDruidIndexers = async (datasources: any, status = HealthStatus.Healthy) => { + const results = await Promise.all(_.map(datasources, (datasource) => getDruidDataourceStatus(datasource["datasource"]))) + const indexers: any = [] + _.forEach(results, (result: any) => { + logger.debug({ result }) + const sourceStatus = _.get(result, "payload.state") == "RUNNING" ? HealthStatus.Healthy : HealthStatus.UnHealthy + logger.debug({ sourceStatus }) + if (sourceStatus == status) { + indexers.push( + { + "type": "druid", + "datasource": _.get(result, "id"), + "status": sourceStatus, + "state": _.get(result, "payload.state") + } + ) + } + }) + return indexers +} + +const restartDruidSupervisors = async (datasourceId: string) => { +const { data } = await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceId}/resume`) +return data; +} + +export const restartDruidIndexers = async (datasources: any) => { + await Promise.all(_.map(datasources, (datasource) => restartDruidSupervisors(datasource["datasource"]))) +} diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts new file mode 100644 index 00000000..2e800212 --- /dev/null +++ b/api-service/src/services/DatasetService.ts @@ -0,0 +1,395 @@ +import _ from "lodash"; +import logger from "../logger"; +import { Dataset } from "../models/Dataset"; +import { DatasetDraft } from "../models/DatasetDraft"; +import { DatasetTransformations } from "../models/Transformation"; +import { DatasetTransformationsDraft } from "../models/TransformationDraft"; +import Model from "sequelize/types/model"; +import { DatasetSourceConfigDraft } from "../models/DatasetSourceConfigDraft"; +import { query, sequelize } from "../connections/databaseConnection"; +import { DatasetSourceConfig } from "../models/DatasetSourceConfig"; +import { ConnectorInstances } from "../models/ConnectorInstances"; +import { DatasourceDraft } from "../models/DatasourceDraft"; +import { executeCommand } from "../connections/commandServiceConnection"; +import Transaction from "sequelize/types/transaction"; +import { DatasetStatus, DatasetType } from "../types/DatasetModels"; +import { Datasource } from "../models/Datasource"; +import { obsrvError } from "../types/ObsrvError"; +import { druidHttpService } from "../connections/druidConnection"; +import { tableGenerator } from "./TableGenerator"; + +class DatasetService { + + getDataset = async (datasetId: string, attributes?: string[], raw = false): Promise => { + return Dataset.findOne({ where: { id: datasetId }, attributes, raw: raw }); + } + + findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { + return Dataset.findAll({ where, attributes, order, raw: true }) + } + + getDuplicateDenormKey = (denormConfig: Record): Array => { + if (denormConfig && _.isArray(_.get(denormConfig, "denorm_fields"))) { + const denormFields = _.get(denormConfig, "denorm_fields") + const denormOutKeys = _.map(denormFields, field => _.get(field, "denorm_out_field")) + const duplicateDenormKeys: Array = _.filter(denormOutKeys, (item: string, index: number) => _.indexOf(denormOutKeys, item) !== index); + return duplicateDenormKeys; + } + return [] + } + + checkDatasetExists = async (dataset_id: string): Promise => { + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); + if (draft === null) { + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); + return !(live === null) + } else { + return true; + } + } + + getDraftDataset = async (dataset_id: string, attributes?: string[]) => { + return DatasetDraft.findOne({ where: { dataset_id }, attributes, raw: true }); + } + + findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { + return DatasetDraft.findAll({ where, attributes, order, raw: true }) + } + + getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { + return DatasetTransformationsDraft.findAll({ where: { dataset_id }, attributes, raw: true }); + } + + getDraftConnectors = async (dataset_id: string, attributes?: string[]) => { + return DatasetSourceConfigDraft.findAll({ where: { dataset_id }, attributes, raw: true }); + } + + getConnectorsV1 = async (dataset_id: string, attributes?: string[]) => { + return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); + } + + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); + } + + getTransformations = async (dataset_id: string, attributes?: string[]) => { + return DatasetTransformations.findAll({ where: { dataset_id }, attributes, raw: true }); + } + + updateDraftDataset = async (draftDataset: Record): Promise> => { + + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); + const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; + logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); + return responseData; + } + + createDraftDataset = async (draftDataset: Record): Promise> => { + + const response = await DatasetDraft.create(draftDataset); + const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: draftDataset.version_key }; + logger.info({ draftDataset, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }); + return responseData; + } + + migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { + const dataset_id = _.get(dataset, "id") + const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); + const transaction = await sequelize.transaction(); + try { + await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); + await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); + await DatasetSourceConfigDraft.destroy({ where: { dataset_id }, transaction }); + await DatasourceDraft.destroy({ where: { dataset_id }, transaction }); + await transaction.commit(); + } catch (err) { + await transaction.rollback(); + throw err; + } + return await this.getDraftDataset(datasetId); + } + + migrateDatasetV1 = async (dataset_id: string, dataset: Record): Promise => { + const status = _.get(dataset, "status") + const draftDataset: Record = { + api_version: "v2", + version_key: Date.now().toString() + } + const dataset_config: any = _.get(dataset, "dataset_config"); + draftDataset["dataset_config"] = { + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } + } + const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] + const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); + draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") + return { + field_key: _.get(config, ["field_key"]), + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", + category: this.getTransformationCategory(section) + }, + mode: _.get(config, ["mode"]) + } + }) + const connectorsFields = ["id", "connector_type", "connector_config"] + const connectors = _.includes([DatasetStatus.Live], status) ? await this.getConnectorsV1(dataset_id, connectorsFields) : await this.getDraftConnectors(dataset_id, connectorsFields); + draftDataset["connectors_config"] = _.map(connectors, (config) => { + return { + id: _.get(config, ["id"]), + connector_id: _.get(config, ["connector_type"]), + connector_config: _.get(config, ["connector_config"]), + version: "v1" + } + }) + draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) + draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["status"] = DatasetStatus.Draft + return draftDataset; + } + + getTransformationCategory = (section: string): string => { + + switch (section) { + case "pii": + return "pii"; + case "additionalFields": + return "derived"; + case "derived": + return "derived"; + default: + return "transform"; + } + } + + createDraftDatasetFromLive = async (dataset: Model) => { + + const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { + draftDataset["dataset_config"] = { + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } + } + const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); + draftDataset["connectors_config"] = _.map(connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); + draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") + return { + field_key: _.get(config, "field_key"), + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, "metadata._transformedFieldDataType") || "string", + category: this.getTransformationCategory(section), + }, + mode: _.get(config, "mode") + } + }) + draftDataset["api_version"] = "v2" + draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) + } else { + const v1connectors = await getV1Connectors(draftDataset.dataset_id); + const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); + draftDataset["connectors_config"] = _.concat(v1connectors, v2connectors) + const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); + draftDataset["transformations_config"] = transformations + } + const denormConfig = _.get(draftDataset, "denorm_config") + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) + if (_.isEmpty(masterDatasets)) { + throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } + } + const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { + const { redis_db, denorm_out_field, denorm_key } = field + let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) + if (!masterConfig) { + masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) + } + if (_.isEmpty(masterConfig)) { + throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", message: `The dependent master dataset is not published`, errCode: "PRECONDITION_REQUIRED", statusCode: 428 } + } + return { denorm_key, denorm_out_field, dataset_id: _.get(masterConfig, "dataset_id") } + }) + draftDataset["denorm_config"] = { ...denormConfig, denorm_fields: updatedDenormFields } + } + draftDataset["version_key"] = Date.now().toString() + draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version + draftDataset["status"] = DatasetStatus.Draft + const result = await DatasetDraft.create(draftDataset); + return _.get(result, "dataValues") + } + + getNextRedisDBIndex = async () => { + return await query("SELECT nextval('redis_db_index')") + } + + deleteDraftDataset = async (dataset: Record) => { + + const { id } = dataset + const transaction = await sequelize.transaction() + try { + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) + await transaction.commit() + } catch (err: any) { + await transaction.rollback() + throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) + } + } + + retireDataset = async (dataset: Record) => { + + const transaction = await sequelize.transaction(); + try { + await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); + await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await transaction.commit(); + await this.deleteDruidSupervisors(dataset); + } catch (err: any) { + await transaction.rollback(); + throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); + } + } + + findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { + return Datasource.findAll({ where, attributes, order, raw: true }) + } + + private deleteDruidSupervisors = async (dataset: Record) => { + + try { + if (dataset.type !== DatasetType.master) { + const datasourceRefs = await Datasource.findAll({ where: { dataset_id: dataset.id }, attributes: ["datasource_ref"], raw: true }) + for (const sourceRefs of datasourceRefs) { + const datasourceRef = _.get(sourceRefs, "datasource_ref") + await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceRef}/terminate`) + logger.info(`Datasource ref ${datasourceRef} deleted from druid`) + } + } + } catch (error: any) { + logger.error({ error: _.get(error, "message"), message: `Failed to delete supervisors for dataset:${dataset.id}` }) + } + } + + publishDataset = async (draftDataset: Record) => { + + const indexingConfig = draftDataset.dataset_config.indexing_config; + const transaction = await sequelize.transaction() + try { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { + await this.createDruidDataSource(draftDataset, transaction); + } + if (indexingConfig.lakehouse_enabled) { + const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); + + if (liveDataset && liveDataset.api_version === "v2") { + await this.updateHudiDataSource(draftDataset, transaction) + } else { + await this.createHudiDataSource(draftDataset, transaction) + } + } + await transaction.commit() + } catch (err: any) { + await transaction.rollback() + throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); + } + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); + + } + + private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { + + const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); + const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); + const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); + _.set(draftDatasource, "ingestion_spec", ingestionSpec) + await DatasourceDraft.create(draftDatasource, { transaction }) + } + + private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + + const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); + const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); + const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); + _.set(draftDatasource, "ingestion_spec", ingestionSpec) + await DatasourceDraft.create(draftDatasource, { transaction }) + } + + private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + + const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); + const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record + const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); + _.set(draftDatasource, "ingestion_spec", ingestionSpec) + await DatasourceDraft.create(draftDatasource, { transaction }) + } + + private createDraftDatasource = (draftDataset: Record, type: string): Record => { + + const datasource = _.join([draftDataset.dataset_id, "events"], "_") + return { + id: _.join([datasource, type], "_"), + datasource: draftDataset.dataset_id, + dataset_id: draftDataset.id, + datasource_ref: datasource, + type + } + } + +} + +export const getLiveDatasetConfigs = async (dataset_id: string) => { + + const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) + const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) + const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV1 = await getV1Connectors(dataset_id) + const connectors = _.concat(connectorsV1,connectorsV2) + + if (!_.isEmpty(transformations)) { + datasetRecord["transformations_config"] = transformations + } + if (!_.isEmpty(connectors)) { + datasetRecord["connectors_config"] = connectors + } + return datasetRecord; +} + +export const getV1Connectors = async (datasetId: string) => { + const v1connectors = await datasetService.getConnectorsV1(datasetId, ["id", "connector_type", "connector_config"]); + const modifiedV1Connectors = _.map(v1connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + return modifiedV1Connectors; +} + +export const datasetService = new DatasetService(); \ No newline at end of file diff --git a/api-service/src/v2/services/DatasetSourceConfigService.ts b/api-service/src/services/DatasetSourceConfigService.ts similarity index 100% rename from api-service/src/v2/services/DatasetSourceConfigService.ts rename to api-service/src/services/DatasetSourceConfigService.ts diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts new file mode 100644 index 00000000..812a5611 --- /dev/null +++ b/api-service/src/services/DatasourceService.ts @@ -0,0 +1,23 @@ +import { Datasource } from "../models/Datasource"; + +export const getDatasourceList = async (datasetId: string, raw = false) => { + const dataSource = await Datasource.findAll({ + where: { + dataset_id: datasetId, + }, + raw: raw + }); + return dataSource +} + + + + + + + + + + + + diff --git a/api-service/src/services/HealthService.ts b/api-service/src/services/HealthService.ts new file mode 100644 index 00000000..c10a9ea6 --- /dev/null +++ b/api-service/src/services/HealthService.ts @@ -0,0 +1,20 @@ +import { Request, Response } from "express" +import { ResponseHandler } from "../helpers/ResponseHandler" + +class HealthService { + + async checkDruidHealth(req: Request, res: Response) { + ResponseHandler.successResponse(req, res, { status: 200, data: {} }) + } + + async checkKafkaHealth() : Promise { + return true + } + + async checkPostgresHealth() : Promise { + return true + } + +} + +export const healthService = new HealthService() \ No newline at end of file diff --git a/api-service/src/v2/services/QueryTemplateService.ts b/api-service/src/services/QueryTemplateService.ts similarity index 100% rename from api-service/src/v2/services/QueryTemplateService.ts rename to api-service/src/services/QueryTemplateService.ts diff --git a/api-service/src/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/services/SchemaGenerateService/ConfigSuggester.ts new file mode 100644 index 00000000..8834129d --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/ConfigSuggester.ts @@ -0,0 +1,61 @@ +import _ from "lodash"; +import { DataSetConfig, DatasetProcessing } from "../../types/ConfigModels"; +import { IngestionConfig, RollupInfo } from "../../types/IngestionModels"; +import { ConflictTypes } from "../../types/SchemaModel"; +import { ingestionConfig } from "../../configs/IngestionConfig"; +import { parseSchemaPath } from "./SchemaGeneratorUtils"; + + +export const defaultLabels = { + dropDuplicates: ["Yes", "No"] +} + +export class ConfigSuggestor { + /** + * Responsiblities : + * 1. Suggest rollup is required or not. - done + * 2. Suggest the dedup property fields. - done + */ + private dataset: string + private rollupInfo: RollupInfo + constructor(dataset: string) { + this.dataset = dataset + this.rollupInfo = {} + } + + public suggestConfig(conflicts: ConflictTypes[], rollupInfo: RollupInfo): DataSetConfig { + this.rollupInfo = rollupInfo + const suggestedConfig = this.analyzeConflicts(conflicts) + return suggestedConfig + } + + private analyzeConflicts(conflicts: ConflictTypes[]): DataSetConfig { + const typeFormatsConflict: ConflictTypes[] = _.filter(conflicts, (o) => !_.isEmpty(o.formats)); + const ingestionConfig: IngestionConfig = this.ingestionConfig() + const processingConfig: DatasetProcessing = this.processingConfig(typeFormatsConflict) + return { "indexConfiguration": ingestionConfig, "processing": processingConfig } + } + + private ingestionConfig(): any { + return { "index": Object.assign(ingestionConfig.indexCol), "rollupSuggestions": this.rollupInfo }; + } + + private processingConfig(conflicts: ConflictTypes[]): any { + let dedupKeys = _.filter(conflicts, (o) => _.upperCase(o.formats.resolution["type"]) === "DEDUP").map(v => v.formats.property) + let matchedDedupFields = [] + const dedupOrderProperty: string = "cardinality" + const dedupOrder: any = "desc" + if (!_.isUndefined(this.rollupInfo.summary)) { + for (const key of Object.keys(this.rollupInfo.summary)) { + if (!this.rollupInfo.summary[key].index) { + for (const dedupKey of dedupKeys) { + if (dedupKey == parseSchemaPath(this.rollupInfo.summary[key].path)) matchedDedupFields.push(this.rollupInfo.summary[key]) + } + } + } + matchedDedupFields = _.orderBy(matchedDedupFields, dedupOrderProperty, dedupOrder) + dedupKeys = _.map(matchedDedupFields, matchedDedupField => parseSchemaPath(matchedDedupField.path)); + } + return { "dedupKeys": dedupKeys, dropDuplicates: defaultLabels.dropDuplicates } + } +} diff --git a/api-service/src/services/SchemaGenerateService/Constants.json b/api-service/src/services/SchemaGenerateService/Constants.json new file mode 100644 index 00000000..ba64c940 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/Constants.json @@ -0,0 +1,20 @@ +{ + "SCHEMA_RESOLUTION_TYPE": { + "OPTIONAL_TYPE": "OPTIONAL_TYPE", + "FORMAT_TYPE": "FORMAT_TYPE", + "DATA_TYPE": "DATA_TYPE", + "INDEX": "INDEX", + "TRANSFORM": "TRANSFORM", + "DROP": "DROP", + "DEDUP": "DEDUP", + "NULL_FIELD": "NULL_FIELD", + "ARRIVAL_FORMAT": "ARRIVAL_FORMAT" + }, + "SEVERITY": { + "CRITICAL": "CRITICAL", + "MEDIUM": "MEDIUM", + "HIGH": "HIGH", + "LOW": "LOW", + "MUST-FIX": "MUST-FIX" + } +} diff --git a/api-service/src/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts new file mode 100644 index 00000000..67bde9e7 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts @@ -0,0 +1,106 @@ +import { inferSchema } from "@jsonhero/schema-infer"; +import httpStatus from "http-status"; +import _ from "lodash"; +import moment from "moment"; +import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException"; + +const DATE_FORMATS = [ + "MM/DD/YYYY","DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", + "DD-MM-YYYY", "MM-DD-YYYY", "MM-DD-YYYY HH:mm:ss", "YYYY/MM/DD HH:mm:ss", + "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", + "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", + "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", + "YYYY-DD-MMTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ss.SSSZ", "DD/MM/YYYYTHH:mm:ss.SSSZ", + "YYYY-DD-MMTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss.SSS", "DD/MM/YYYYTHH:mm:ss.SSS", + "MM-DD-YYYYTHH:mm:ss.SSSZ", "DD-MM-YYYYTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ssZ", + "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", + "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", + "YYYY/MM/DDTHH:mm:ss", "DD/MM/YYYYTHH:mm:ss", "DD-MM-YYYY HH:mm:ss.SSSZ", "YYYY-MM-DD HH:mm:ss.SSSZ", + "YYYY-DD-MM HH:mm:ss.SSSZ", "YYYY/MM/DD HH:mm:ss.SSSZ", "DD/MM/YYYY HH:mm:ss.SSSZ", + "MM-DD-YYYY HH:mm:ss.SSSZ", "DD-MM-YYYY HH:mm:ssZ", "YYYY-MM-DD HH:mm:ssZ", "YYYY-DD-MM HH:mm:ssZ", + "YYYY/MM/DD HH:mm:ssZ", "DD/MM/YYYY HH:mm:ssZ", "MM-DD-YYYY HH:mm:ssZ", "DD-MM-YYYYTHH:mm:ss.SSSSSSZ", + "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", "YYYY-DD-MMTHH:mm:ss.SSSSSSZ", "YYYY/MM/DDTHH:mm:ss.SSSSSSZ", + "DD/MM/YYYYTHH:mm:ss.SSSSSSZ", "MM-DD-YYYYTHH:mm:ss.SSSSSSZ", "DD/MM/YYYYTHH:mm:ss.SSSSSS", + "YYYY-DD-MMTHH:mm:ss.SSSSSS", "YYYY/MM/DDTHH:mm:ss.SSSSSS", "YYYY-MM-DDTHH:mm:ss.SSSSSS", + "MM-DD-YYYYTHH:mm:ss.SSSSSS", "DD-MM-YYYYTHH:mm:ss.SSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSS", + "YYYY-MM-DD HH:mm:ss.SSSSSS", "YYYY-DD-MM HH:mm:ss.SSSSSS", "YYYY/MM/DD HH:mm:ss.SSSSSS", + "DD/MM/YYYY HH:mm:ss.SSSSSS", "MM-DD-YYYY HH:mm:ss.SSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSZ", + "YYYY-MM-DDTHH:mm:ss.SSSSSSSSSZ", "YYYY-DD-MMTHH:mm:ss.SSSSSSSSSZ", "YYYY/MM/DDTHH:mm:ss.SSSSSSSSSZ", + "DD/MM/YYYYTHH:mm:ss.SSSSSSSSSZ", "MM-DD-YYYYTHH:mm:ss.SSSSSSSSSZ", "DD/MM/YYYYTHH:mm:ss.SSSSSSSSS", + "YYYY-DD-MMTHH:mm:ss.SSSSSSSSS", "YYYY/MM/DDTHH:mm:ss.SSSSSSSSS", "YYYY-MM-DDTHH:mm:ss.SSSSSSSSS", + "MM-DD-YYYYTHH:mm:ss.SSSSSSSSS", "DD-MM-YYYYTHH:mm:ss.SSSSSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSSSS", + "YYYY-MM-DD HH:mm:ss.SSSSSSSSS", "YYYY-DD-MM HH:mm:ss.SSSSSSSSS", "YYYY/MM/DD HH:mm:ss.SSSSSSSSS", + "DD/MM/YYYY HH:mm:ss.SSSSSSSSS", "MM-DD-YYYY HH:mm:ss.SSSSSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSSSSZ", + "DD-MM-YYYYTHH:mm:ss.SSSSSSSSSZ", +]; + +export class SchemaInference { + + public inferSchema(sample: any) { + const schema = _.map(sample, (value): any => this.validateEpoch(inferSchema(value).toJSONSchema({ includeSchema: true }), value, "properties")) + return schema + } + + public inferBatchSchema(sample: Map[], extractionKey: string) { + return _.flatMap(sample, (value) => { + if (extractionKey) { + const extracted = _.get(value, extractionKey); + if (extracted) { + return this.inferSchema(extracted); + } else { + throw new SchemaGenerationException("Unable to extract the batch data.", httpStatus.BAD_REQUEST); + } + } else { + throw new SchemaGenerationException("Extraction key not found.", httpStatus.BAD_REQUEST); + } + }) + } + + private validateEpoch(schema: any, sample: any, path: any) { + Object.entries(sample).map(([key, value]) => { + if (value && typeof value == "object") { + this.validateEpoch(schema, value, `${path}.${key}.properties`) + } + const { isValidTimestamp, type } = this.isValidTimestamp(value); + const format = _.get(schema, `${path}.${key}.format`); + if (isValidTimestamp) { + _.set(schema, `${path}.${key}.format`, type) + } + else if(format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) { + _.unset(schema, `${path}.${key}.format`) + } + }); + return schema + } + + isValidTimestamp(value: any) { + const dataType = typeof value; + const epochRegex = /^\d+$/ig; + switch (dataType) { + case "string": + if(epochRegex.test(value)){ + const parsedValue = parseInt(value, 10); + // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds + return { + isValidTimestamp: parsedValue >= 946684800 && moment(parsedValue).isValid(), + type: "epoch" + } + } else return { + isValidTimestamp: moment(value, DATE_FORMATS, true).isValid(), + type: "date-time" + } + case "number": + // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds + return { + isValidTimestamp: value >= 946684800 && moment(value).isValid(), + type: "epoch" + }; + default: + return { + isValidTimestamp: false, + type: "" + }; + } + } +} \ No newline at end of file diff --git a/api-service/src/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/services/SchemaGenerateService/SchemaAnalyser.ts new file mode 100644 index 00000000..a3a24b1d --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaAnalyser.ts @@ -0,0 +1,193 @@ +import _ from "lodash"; +import { Conflict, ConflictTypes, FlattenSchema, Occurance } from "../../types/SchemaModel"; +import constants from "./Constants.json"; +export class SchemaAnalyser { + private transformationCols = ["email", "creditcard", "ipv4", "ipv6"] + private dateFormatCols = ["date", "date-time"] + private cardinalCols = ["uuid"] + + private schemas: Map[]; + private minimumSchemas: number = 1 + + + constructor(schemas: Map[]) { + this.schemas = schemas; + } + + /** + * Method to analyse the schema + */ + public analyseSchema(): ConflictTypes[] { + return this.findConflicts() + } + + private findConflicts(): ConflictTypes[] { + const result: FlattenSchema[] = _.flatten(this.schemas.map(element => { + return this.flattenSchema(new Map(Object.entries(element))); + })) + const conflicts = Object.entries(_.groupBy(result, "path")).map(([, value]) => { + return this.getSchemaConflictTypes(this.getOccurance(value)) + }) + return _.filter(conflicts, obj => (!_.isEmpty(obj.schema) || !_.isEmpty(obj.required) || !_.isEmpty(obj.formats))) + } + + /** + * Retruns possible conflicts types + * 1. data type conflicts + * 2. Data format conflicts + * 3. Optional property conflicts + */ + private getSchemaConflictTypes(occuranceObj: Occurance): ConflictTypes { + const absolutePath = _.head(_.keysIn(occuranceObj.absolutePath)) + const updatedPath = absolutePath ? _.replace(absolutePath, "$.", "") : ""; + let schemaConflicts = this.findDataTypeConflicts(occuranceObj,) + const requiredConflicts = (_.size(this.schemas) > this.minimumSchemas) ? this.findOptionalPropConflicts(occuranceObj) : {} + const formatConflict = this.findFormatConflicts(occuranceObj) + if (_.size(_.keys(schemaConflicts)) > 0) { + schemaConflicts = { ...schemaConflicts, path: updatedPath } + } + return { "schema": schemaConflicts, "required": requiredConflicts, "formats": formatConflict, "absolutePath": updatedPath } + } + + /** + * Method to get the data type conflicts + */ + private findDataTypeConflicts(occurance: Occurance): Conflict { + if (_.includes(_.keys(occurance.dataType), "null") && _.size(occurance.dataType) === 1) { + return { + type: constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD, + // Should be used only to return the name of field instead of path + // property: Object.keys(occurance.property)[0], + property: _.replace(Object.keys(occurance.path)[0], "$.", ""), + conflicts: occurance.dataType, + resolution: { "value": occurance.dataType, "type": constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD }, + values: _.keys(occurance.dataType), + severity: constants.SEVERITY["MUST-FIX"], + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } + const minimumOccurance: number = 1 + if (_.size(occurance.dataType) > minimumOccurance) { + const isUnresolvable: boolean = _.uniq(_.values(occurance.dataType)).length === 1; + const highestValueKey = !isUnresolvable ? Object.keys(occurance.dataType).reduce((a, b) => occurance.dataType[a] > occurance.dataType[b] ? a : b) : undefined + return { + type: constants.SCHEMA_RESOLUTION_TYPE.DATA_TYPE, + property: _.replace(Object.keys(occurance.path)[0], "$.", ""), + conflicts: occurance.dataType, + resolution: { "value": highestValueKey, "type": constants.SCHEMA_RESOLUTION_TYPE.DATA_TYPE }, + values: _.without(_.keys(occurance.dataType), "null"), + severity: constants.SEVERITY["MUST-FIX"], + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } else { return {} } + } + + /** + * Method to get the format type conflicts + * + */ + private findFormatConflicts(occurance: Occurance): Conflict { + const filteredFormat = _.omit(occurance.format, "undefined") + const formats = _.concat(this.transformationCols, this.dateFormatCols, this.cardinalCols, ["uri"]); + if (!_.isEmpty(filteredFormat)) { + const formatName = _.filter(formats, f => _.has(filteredFormat, f)); + const suggestedFormat = this.idenfityFormat(formatName[0]) + return { + type: formatName[0], + property: _.replace(Object.keys(occurance.path)[0], "$.", ""), + conflicts: filteredFormat, + resolution: { "value": formatName, "type": suggestedFormat.type }, + values: _.keys(filteredFormat), + severity: suggestedFormat.severity || "", + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } else { return {} } + + } + + + private idenfityFormat(value: string) { + if (_.includes(this.transformationCols, value)) { + return { type: "TRANSFORMATION", "severity": "LOW" }; + } else if (_.includes(this.dateFormatCols, value)) { + return { type: "INDEX", "severity": "LOW" }; + } else if (_.includes(this.cardinalCols, value)) { + return { type: "DEDUP", "severity": "LOW" }; + } else { + return {}; + } + } + + /** + * + */ + private findOptionalPropConflicts(occurance: Occurance): Conflict { + const maxOccurance: number = 1 + const requiredCount = _.map(occurance.property, (value) => { + return value + })[0] + + const highestValueKey = Boolean(Object.keys(occurance.isRequired).reduce((a, b) => occurance.isRequired[a] > occurance.isRequired[b] ? a : b)) + const isPropertyRequired = requiredCount <= maxOccurance ? false : true + if (highestValueKey != isPropertyRequired) { + return { + type: constants.SCHEMA_RESOLUTION_TYPE.OPTIONAL_TYPE, + property: Object.keys(occurance.property)[0], + conflicts: occurance.property, + resolution: { "value": (isPropertyRequired), "type": "OPTIONAL" }, + values: [true, false], + severity: "MEDIUM", + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } + else { return {} } + + } + + /** + * + * Method to get the occurance of the given key from the given object + */ + private getOccurance(arrayOfObjects: object[]): Occurance { + const result = _(arrayOfObjects).flatMap(obj => _.toPairs(obj)).groupBy(([key]) => key) + .mapValues(group => _.countBy(group, ([, value]) => value)).value(); + return { property: result.property, dataType: result.dataType, isRequired: result.isRequired, path: result.path, absolutePath: result.absolutePath, format: result.formate }; + } + + /** + * Method to iterate over the schema object in a recursive and flatten the required properties + */ + public flattenSchema(sample: Map): FlattenSchema[] { + const array: any[] = []; + const recursive = (data: any, path: string, requiredProps: string[], schemaPath: string) => { + _.map(data, (value, key) => { + let isMultipleTypes = ""; + if (_.has(value, "anyOf")) isMultipleTypes = "anyOf"; + if (_.has(value, "oneOf")) isMultipleTypes = "oneOf"; + if (_.isPlainObject(value) && (_.has(value, "properties"))) { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + recursive(value["properties"], `${path}.${key}`, value["required"], `${schemaPath}.properties.${key}`); + } else if (_.isPlainObject(value)) { + if (value.type === "array") { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + if (_.has(value, "items") && _.has(value["items"], "properties")) { + recursive(value["items"]["properties"], `${path}.${key}[*]`, value["items"]["required"], `${schemaPath}.properties.${key}.items`); + } + } else if (isMultipleTypes != "") { + array.push(this._flattenSchema(key, value[isMultipleTypes][0].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + array.push(this._flattenSchema(key, value[isMultipleTypes][1].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + } else { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + } + } + }) + } + recursive(sample.get("properties"), "$", sample.get("required"), "$") + return array + } + + private _flattenSchema(expr: string, objType: string, isRequired: boolean, path: string, schemaPath: string, formate: string): FlattenSchema { + return { "property": expr, "dataType": objType, "isRequired": isRequired, "path": path, "absolutePath": schemaPath, "formate": formate } + } + +} diff --git a/api-service/src/services/SchemaGenerateService/SchemaArrayValidator.ts b/api-service/src/services/SchemaGenerateService/SchemaArrayValidator.ts new file mode 100644 index 00000000..4bd19c91 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaArrayValidator.ts @@ -0,0 +1,52 @@ +import _, { isUndefined } from "lodash"; + +export class SchemaArrayValidator { + public validate(schemas: any) { + _.map(schemas, (schema: any, index: number) => { + Object.entries(schema).map(([schemaKey, schemaValue]) => { + if (typeof schemaValue === "object") { + this.handleNestedObject(index, `${schemaKey}`, schemaValue, schemas); + } + }); + }); + return schemas + } + + private checkForInvalidArray(value: any) { + if (_.has(value, "items") && _.has(value, "properties")) + _.unset(value, "properties"); + } + + private handleNestedObject(index: any, path: string, value: any, schemas: any) { + Object.entries(value).map(([nestedKey, nestedValue]: any) => { + if (typeof nestedValue === "object") { + this.handleNestedObject(index, `${path}.${nestedKey}`, nestedValue, schemas) + } else if (nestedValue.type === "array" && (nestedValue.items != false)) { + this.checkForInvalidArray(nestedValue); + let isValidArray = true; + if(_.isEqual(_.get(schemas[0], `${path}.${nestedKey}.type`), _.get(schemas[index], `${path}.${nestedKey}.type`))) { + isValidArray = _.isEqual( + _.get(schemas[0], `${path}.${nestedKey}.items`), + _.get(schemas[index], `${path}.${nestedKey}.items`) + ) + } + if (!isValidArray) { + this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) + } + } else if (nestedValue.type === "array" && (nestedValue.items == false)) { + this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) + } + }) + } + + private deleteItemsAndSetAdditionalProperties(schemas: any, path: string) { + _.map((schemas), (schema: any) => { + if (!isUndefined(_.get(schema, path))) { + _.unset(schema, `${path}`) + _.set(schema, `${path}.type`, "array"); + _.set(schema, `${path}.additionalProperties`, false); + } + }); + } + +} diff --git a/api-service/src/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts b/api-service/src/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts new file mode 100644 index 00000000..23dee0cb --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts @@ -0,0 +1,50 @@ +import { SchemaMerger } from "./SchemaMerger"; +import _ from "lodash"; +import { generateRollupSummary, generateSchemaPath, updateCardinalColumns } from "./SchemaGeneratorUtils"; +import { FieldSchema, UniqueValues } from "../../types/SchemaModel"; +export class SchemaCardinalityAnalyser { + private sampleData: Record[] + private sampleSchema: Map[] + private schemaMerger: SchemaMerger + private mergedSchema: FieldSchema + private uniqueValues: UniqueValues + constructor(sampleData: Record[], sampleSchema: Map[]) { + this.sampleData = sampleData + this.sampleSchema = sampleSchema + this.schemaMerger = new SchemaMerger() + this.mergedSchema = this.schemaMerger.mergeSchema(this.sampleSchema) || {} + this.uniqueValues = {} + } + + public analyse() { + this.generateUniqueValues(this.sampleData, this.mergedSchema) + return generateRollupSummary(this.uniqueValues) + } + + private generateUniqueValues(data: any, schema: FieldSchema, currentPath: string = "") { + if (!_.isUndefined(data) && !_.isUndefined(schema) && _.isArray(data)) { + for (const item of data) { + if (schema?.type === "object" && !_.isUndefined(schema.properties)) { + this.identifyCardinalColumns(item, schema.properties, currentPath); + } else if (schema?.type === "array" && !_.isUndefined(schema.items)) { + this.generateUniqueValues(item, schema.items, currentPath); + } + } + } + } + + private identifyCardinalColumns(data: Record, schema: any, currentPath: string) { + if (!_.isUndefined(data) && !_.isUndefined(schema)) { + for (const key of Object.keys(schema)) { + const fieldSchema = schema[key] + const path = generateSchemaPath(currentPath, key) + if (fieldSchema?.type === "object") { + this.identifyCardinalColumns(data[key], fieldSchema.properties, path); + } else if (fieldSchema?.type === "array") { + this.generateUniqueValues(data[key], fieldSchema.items || {}, `${path}[*]`); + } + updateCardinalColumns(data, fieldSchema, currentPath, key, this.uniqueValues) + } + } + } +} diff --git a/api-service/src/services/SchemaGenerateService/SchemaGeneratorUtils.ts b/api-service/src/services/SchemaGenerateService/SchemaGeneratorUtils.ts new file mode 100644 index 00000000..99f15ea2 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaGeneratorUtils.ts @@ -0,0 +1,50 @@ +import _ from "lodash"; +import { config } from "../../configs/Config"; +import { UniqueValues, FieldSchema, RollupSummary } from "../../types/SchemaModel"; + + +export const generateRollupSummary = (uniqueValues: UniqueValues) => { + const summary: RollupSummary = {}; + Object.entries(uniqueValues).map(([field, value]) => { + const data: Record = {}; + _.map(value, (item: string) => { + if (!_.has(data, [field, item])) _.set(data, [field, item], 1); + else data[field][item] += 1; + }); + const resultData: Record = {}; + _.map(_.keys(data), (path: string) => { + Object.entries(data[path]).map(([, value]: any) => { + const totalValue = _.sum(_.values(data[path])); + const ratio = Math.round((value / totalValue) * 100); + if (!_.has(resultData, path)) _.set(resultData, path, ratio); + else if (ratio > _.get(resultData, path)) + _.set(resultData, path, ratio); + }); + const fieldName = parseSchemaPath(path); + summary[fieldName] = { + path: `$.${path}`, + cardinality: 100 - _.get(resultData, path), + index: _.get(resultData, path) >= config.rollup_ratio, + }; + }); + }); + return { summary }; +}; + +export const updateCardinalColumns = (data: Record, fieldSchema: FieldSchema, path: string, key: string, uniqueValues: UniqueValues) => { + const schemaPath = generateSchemaPath(path, key); + if (_.includes(config.unique_formats, fieldSchema.format)) { + const value = _.get(data, key); + if (!_.has(uniqueValues, [schemaPath])) uniqueValues[schemaPath] = [value]; + else uniqueValues[schemaPath].push(value); + } + return uniqueValues; +}; + +export const generateSchemaPath = (path: string, key: string) => { + return (path ? `${path}.properties.${key}` : key).replace(/\[\*\]/gi, ".items"); +}; + +export const parseSchemaPath = (path: string) => { + return path.replace("$.", "").replace(/.properties/gi, "").replace(/.items/gi, "[*]"); +}; diff --git a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts new file mode 100644 index 00000000..45d76c9c --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts @@ -0,0 +1,168 @@ +import _ from "lodash"; +import { ConflictTypes, SuggestionsTemplate } from "../../types/SchemaModel"; +import { SchemaMerger } from "./SchemaMerger"; +import { SchemaAnalyser } from "./SchemaAnalyser"; +import DataMappings from "./SchemaMapping.json"; + +export const dataMappingPaths = { + "string": "text.store_format.string.jsonSchema", + "date-time": "text.store_format.date-time.jsonSchema", + "date": "text.store_format.date.jsonSchema", + "boolean": "boolean.store_format.boolean.jsonSchema", + "integer": "number.store_format.integer.jsonSchema", + "number": "number.store_format.number.jsonSchema", + "object": "object.store_format.object.jsonSchema", + "array": "array.store_format.array.jsonSchema", +} + +export class SchemaHandler { + private typeToMethod = { + required: this.updateRequiredProp.bind(this), + datatype: this.updateDataTypes.bind(this), + suggestions: this.setSuggestions.bind(this), + setNulltype: this.setNulltype.bind(this) + } + + public process(schema: Map[]): ConflictTypes[] { + const schemaAnalyser = new SchemaAnalyser(schema) + return schemaAnalyser.analyseSchema() + } + + public merge(schema: any): any { + const schemaMerger = new SchemaMerger() + return schemaMerger.mergeSchema(schema) + } + + public update(schema: any, property: any, type: keyof typeof this.typeToMethod) { + const method = this.typeToMethod[type]; + return method(schema, property); + } + + private updateDataTypes(schema: any, conflict: ConflictTypes): any { + const { absolutePath, schema: { resolution } } = conflict; + return _.set(schema, `${absolutePath}`, { + ...schema[absolutePath], + ...{ + type: resolution.value, + oneof: conflict.schema.values.map(key => ({ type: key })), + } + }); + } + + private setNulltype(schema: any, conflict: ConflictTypes): any { + const { absolutePath } = conflict; + const dataTypes: any = []; + _.forEach(DataMappings, (valueItem) => { + _.forEach(_.get(valueItem, "store_format"), (subValue) => { + if (!_.find(dataTypes, ["type", subValue["jsonSchema"]])) + dataTypes.push({ type: subValue["jsonSchema"] }) + }) + }); + const arrivalDataTypes: any = _.keys(DataMappings).map((key: any) => ({ type: key })); + + _.set(schema, `${absolutePath}.type`, "null"); + _.set(schema, `${absolutePath}.arrivalOneOf`, arrivalDataTypes); + return _.set(schema, `${absolutePath}.oneof`, dataTypes); + } + + private updateRequiredProp(schema: any, value: ConflictTypes): any { + const absolutePath = value.absolutePath.replace(value.required.property, value.required.property.replace(".", "$")) + const subStringArray: string[] = _.split(absolutePath, "."); + const subString: string = _.join(_.slice(subStringArray, 0, subStringArray.length - 2), "."); + const path: string = _.isEmpty(subString) ? "required" : `${subString}.required` + const requiredList: string[] = _.get(schema, path) + const newProperty: string = value.required.property + value.required.resolution.value ? _.concat(_.get(schema, path), value.required.property) : _.pull(requiredList, newProperty) + return _.set(schema, path, _.uniq(requiredList)) + } + + private getArrivalSuggestions(schema: any, fieldData: any, property: any, type: string) { + const arrivalSuggestions: any = []; + const types = _.get(fieldData, type); + types && types.map((item: any) => { + const storeFormat = _.get(dataMappingPaths, item.type); + arrivalSuggestions.push({ type: _.first(storeFormat.split(".")) }); + }) + if (arrivalSuggestions.length > 0) + _.set(schema, `${property}.arrivalOneOf`, arrivalSuggestions); + return; + } + + private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { + const types = _.get(fieldData, type); + types && types.map((item: any) => { + const storeFormat = _.get(dataMappingPaths, item.type); + _.set(schema, `${property}.arrival_format`, _.first(storeFormat.split("."))); + _.set(schema, `${property}.data_type`, _.get(DataMappings, storeFormat)); + }) + return; + } + + private setSuggestions(schema: any, suggestedTemplate: SuggestionsTemplate[]): any { + suggestedTemplate && suggestedTemplate.map(({ property, suggestions }) => { + const fieldData = _.get(schema, property); + _.set(schema, `${property}.suggestions`, suggestions); + const arrivalConflictExists = _.filter(suggestions, (suggestion) => _.has(suggestion, "arrivalConflict")); + switch (true) { + // Add arrival conflicts if there is arrival conflict in suggestions + case _.has(fieldData, "oneof") && arrivalConflictExists.length > 0: + return this.getArrivalSuggestions(schema, fieldData, property, "oneof") + // Add arrival type if there are no arrival type conflicts + case arrivalConflictExists.length === 0: + return this.getArrivalFormat(schema, fieldData, property, "oneof") + default: + break; + } + }); + return schema; + } + + private updateStoreType(schemaValue: any, dataType: string): any { + const storeFormat = _.get(dataMappingPaths, dataType); + if (storeFormat) { + _.set(schemaValue, "arrival_format", _.first(storeFormat.split("."))); + if (!_.get(schemaValue, "data_type")) _.set(schemaValue, "data_type", _.get(DataMappings, storeFormat)); + } + } + + private checkForInvalidArray(value: any) { + if (_.has(value, "items") && _.has(value, "properties")) + _.unset(value, "properties"); + } + + private updateMappings(schema: Map) { + const recursive = (data: any) => { + _.map(data, (value) => { + if (_.isPlainObject(value)) { + if ((_.has(value, "properties"))) { + recursive(value["properties"]); + } + if (value.type === "array") { + if (_.has(value, "items") && _.has(value["items"], "properties")) { + recursive(value["items"]["properties"]); + } + if (_.has(value, "items") && _.has(value, "properties")) + this.checkForInvalidArray(value); + this.updateStoreType(value, _.get(value, "type")); + } else { + if (_.get(value, "type") === "string" && ["date-time", "date"].includes(_.get(value, "format"))) { + this.updateStoreType(value, _.get(value, "format")); + } + else { + this.updateStoreType(value, _.get(value, "type")); + } + } + } else { + this.updateStoreType(value, _.get(value, "type")); + } + }) + } + recursive(_.get(schema, "properties")) + } + + public mapDataTypes(schema: any,): any { + this.updateMappings(schema); + return schema; + } + +} diff --git a/api-service/src/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json new file mode 100644 index 00000000..19843cf5 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaMapping.json @@ -0,0 +1,103 @@ +{ + "text": { + "arrival_format": ["string"], + "store_format": { + "string": { + "jsonSchema": "string", + "datasource": "string" + }, + "date-time": { + "jsonSchema": "string", + "datasource": "string" + }, + "date": { + "jsonSchema": "string", + "datasource": "string" + }, + "boolean": { + "jsonSchema": "string", + "datasource": "boolean" + }, + "epoch": { + "jsonSchema": "string", + "datasource": "integer" + }, + "long": { + "jsonSchema": "string", + "datasource": "long" + }, + "double": { + "jsonSchema": "string", + "datasource": "double" + }, + "bigdecimal": { + "jsonSchema": "string", + "datasource": "double" + }, + "integer": { + "jsonSchema": "string", + "datasource": "long" + } + } + }, + "number": { + "arrival_format": ["number", "integer"], + "store_format": { + "integer": { + "jsonSchema": "integer", + "datasource": "long" + }, + "float": { + "jsonSchema": "number", + "datasource": "double" + }, + "long": { + "jsonSchema": "integer", + "datasource": "long" + }, + "double": { + "jsonSchema": "number", + "datasource": "double" + }, + "bigdecimal": { + "jsonSchema": "number", + "datasource": "double" + }, + "epoch":{ + "jsonSchema": "integer", + "datasource": "long" + }, + "number": { + "jsonSchema": "number", + "datasource": "double" + } + } + }, + "object": { + "arrival_format": ["object"], + "store_format": { + "object": { + "jsonSchema": "object", + "datasource": "json" + } + } + }, + "array": { + "arrival_format": ["array"], + "store_format": { + "array": { + "jsonSchema": "array", + "datasource": "array" + } + } + }, + "boolean": { + "arrival_format": ["boolean"], + "store_format": { + "boolean": { + "jsonSchema": "boolean", + "datasource": "boolean" + } + } + } +} diff --git a/api-service/src/services/SchemaGenerateService/SchemaMerger.ts b/api-service/src/services/SchemaGenerateService/SchemaMerger.ts new file mode 100644 index 00000000..a35cc237 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SchemaMerger.ts @@ -0,0 +1,18 @@ +import * as _ from "lodash"; + +export class SchemaMerger { + + mergeSchema(schema: Map[]): any { + try { + let data = {}; + _.map(schema, (item: any) => { + data = _.merge(data, item) + }); + return data; + } + catch (error) { + console.log(error) + } + } + +} \ No newline at end of file diff --git a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts new file mode 100644 index 00000000..fcaba4a5 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts @@ -0,0 +1,74 @@ +import _ from "lodash" +import { Conflict, ConflictTypes, Suggestion, SuggestionsTemplate } from "../../types/SchemaModel" +import constants from "./Constants.json" +import { SchemaSuggestionTemplate } from "./Template" + +export class SuggestionTemplate { + + public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { + return _.map(sample, (value) => { + const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) + const requiredSuggestions = this.getRequiredMessageTemplate(value.required) + const formatSuggestions = this.getPropertyFormatTemplate(value.formats) + return { + "property": value.absolutePath, + "suggestions": _.reject([dataTypeSuggestions, requiredSuggestions, formatSuggestions], _.isEmpty) + } + }) + } + + private getSchemaMessageTemplate(object: Conflict, arrival_format: boolean = false): Suggestion | [] { + if (_.isEmpty(object)) return {} + let message, arrival_format_message, advice; + if (object.type === constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD) { + message = SchemaSuggestionTemplate.getSchemaNullTypeMessage(object.conflicts, object.property); + advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.ADVICE; + } else { + const { conflictMessage, arrivalFormatMessage } = SchemaSuggestionTemplate.getSchemaDataTypeMessage(object.conflicts, object.property); + message = conflictMessage; + arrival_format_message = arrivalFormatMessage; + advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.ADVICE; + } + + const suggestion = { + message: message, + advice, + resolutionType: object.resolution["type"], + severity: object.severity, + path: object.path + }; + if (arrival_format && arrival_format_message) { + suggestion["message"] = arrival_format_message; + suggestion["arrivalConflict"] = true; + suggestion["resolutionType"] = constants.SCHEMA_RESOLUTION_TYPE.ARRIVAL_FORMAT; + } else if(arrival_format && !arrival_format_message) + return []; + return suggestion; + } + + + public getPropertyFormatTemplate(object: Conflict): Suggestion { + if (_.isEmpty(object)) return {} + const conflictMessage = SchemaSuggestionTemplate.getSchemaFormatMessage(object.conflicts, object.property) + const adviceObj = SchemaSuggestionTemplate.getSchemaFormatAdvice(object.conflicts) + return { + message: conflictMessage, + advice: adviceObj.advice, + resolutionType: object.resolution["type"], + severity: object.severity, + path: object.path + } + } + + public getRequiredMessageTemplate(object: Conflict): Suggestion { + if (_.isEmpty(object)) return {} + const conflictMessage = SchemaSuggestionTemplate.getSchemaRequiredTypeMessage(object.conflicts, object.property) + return { + message: conflictMessage, + advice: SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.OPTIONAL_PROPERTY.ADVICE, + resolutionType: object.resolution["type"], + severity: object.severity, + path: object.path + } + } +} diff --git a/api-service/src/services/SchemaGenerateService/Template.ts b/api-service/src/services/SchemaGenerateService/Template.ts new file mode 100644 index 00000000..9beb3a46 --- /dev/null +++ b/api-service/src/services/SchemaGenerateService/Template.ts @@ -0,0 +1,107 @@ +import _ from "lodash"; +import { Conflict } from "../../types/SchemaModel"; +import constants from "./Constants.json"; +import { dataMappingPaths } from "./../SchemaGenerateService/SchemaHandler"; + +export const SchemaSuggestionTemplate = { + TEMPLATES: { + SCHEMA_SUGGESTION: { + CREATE: { + OPTIONAL_PROPERTY: { + MESSAGE: "Conflict in the Schema Generation", + ADVICE: "The Property looks to be Optional. System has updated the property schema to optional", + SEVERITY: constants.SEVERITY.MEDIUM + }, + DATATYPE_PROPERTY: { + MESSAGE: "Conflict in the Schema Generation", + ADVICE: "System can choose highest occurance property or last appeared object property", + SEVERITY: constants.SEVERITY["MUST-FIX"] + }, + NULL_TYPE_PROPERTY: { + MESSAGE: "Conflict in the Schema Generation", + ADVICE: "The Property has a 'null' data type. 'null' is not a valid data type. Please review and update the property schema accordingly.", + SEVERITY: constants.SEVERITY["MUST-FIX"] + }, + FORMAT_PROPERTY: { + MESSAGE: "The Property", + DATE_ADVICE: { + MESSAGE: "The System can index all data on this column", + SEVERITY: constants.SEVERITY.HIGH + }, + DATETIME_ADVICE: { + MESSAGE: "The System can index all data on this column", + SEVERITY: constants.SEVERITY.LOW + }, + UUID_ADVICE: { + MESSAGE: "Suggest to not to index the high cardinal columns", + SEVERITY: constants.SEVERITY.LOW + }, + IPV4_ADVICE: { + MESSAGE: "Suggest to Mask the Personal Information", + SEVERITY: constants.SEVERITY.LOW + }, + IPV6_ADVICE: { + MESSAGE: "Suggest to Mask the Personal Information", + SEVERITY: constants.SEVERITY.LOW + }, + EMAIL_ADVICE: { + MESSAGE: "Suggest to Mask the Personal Information", + SEVERITY: constants.SEVERITY.LOW + } + } + }, + UPDATE: { + REQUIRED_PROPERTY: { + ADVICE: "Might Required to reprocess the existing data" + }, + DATATYPE_PROPERTY: { + ADVICE: "Might Required to reprocess the existing data" + } + } + + } + }, + + getSchemaDataTypeMessage(conflicts: Conflict, property: string) { + const updatedConflicts: any = {}; + _.map((conflicts), (value, key) => { + const path: any = _.get(dataMappingPaths, key); + const types = _.split(path, "."); + if (path && _.size(types) > 0 && !_.keys(updatedConflicts).includes(types[0])) { + updatedConflicts[types[0]] = value; + } + }); + const response: Record = { + conflictMessage: _.template( + `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.MESSAGE} at property: '${property}'. The property type <% _.map(conflicts, (value, key, list) => { %><%= key %>: <%= value %> time(s)<%= _.last(list) === value ? '' : ', ' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts }), + arrivalFormatMessage: null, + }; + if (_.keys(updatedConflicts).length > 1) { + _.set(response, "arrivalFormatMessage", _.template(`${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.MESSAGE} at property: '${property}'. The property type <% _.map(conflicts, (value, key, list) => { %><%= key %>: <%= value %> time(s)<%= _.last(list) === value ? '' : ', ' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts: updatedConflicts })); + } + return response; + }, + + getSchemaNullTypeMessage(conflicts: Conflict, property: string): string { + return `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.MESSAGE} at property: '${property}'. ${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.ADVICE}`; + }, + + getSchemaFormatMessage(conflicts: Conflict, property: string): string { + const conflictKey = _.keys(conflicts)[0]; + return `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.FORMAT_PROPERTY.MESSAGE} '${property}' appears to be '${conflictKey}' format type.`; + }, + + getSchemaRequiredTypeMessage(conflicts: Conflict, property: string): string { + const message = _.template( + `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.OPTIONAL_PROPERTY.MESSAGE} at property: '${property}'. The property <% _.map(conflicts, (value, key, list) => { %><%= key %>: only <%= value %> time(s) appeared <%= _.last(list) === value ? '' : '' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts }); + return message + }, + + getSchemaFormatAdvice(conflicts: Conflict): any { + const conflictKey = `${_.upperCase(_.replace(_.keys(conflicts)[0], "-", ""))}_ADVICE`.replace(/\s/g, "") + return { + "advice": _.get(this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.FORMAT_PROPERTY, `${conflictKey}.MESSAGE`), + } + } + +} diff --git a/api-service/src/services/SystemConfig.ts b/api-service/src/services/SystemConfig.ts new file mode 100644 index 00000000..47404ed7 --- /dev/null +++ b/api-service/src/services/SystemConfig.ts @@ -0,0 +1,22 @@ + + +const defaultThresholds: any = { + "processing": { + "avgProcessingSpeedInSec": 300, + "validationFailuresCount": 5, + "dedupFailuresCount": 5, + "denormFailureCount": 5, + "transformFailureCount": 5 + }, + "query": { + "avgQueryReponseTimeInSec": 5, + "queriesFailed": 10 + } +} +export const SystemConfig = { + + getThresholds: (category: string) => { + return Promise.resolve(defaultThresholds[category]) + } + +} \ No newline at end of file diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts new file mode 100644 index 00000000..e22714f6 --- /dev/null +++ b/api-service/src/services/TableGenerator.ts @@ -0,0 +1,298 @@ +import _ from "lodash"; +import { rawIngestionSpecDefaults } from "../configs/IngestionConfig"; +import { datasetService } from "./DatasetService"; + +class BaseTableGenerator { + + /** + * Method to flatten a json schema - extract all properties using jsonpath notation + * + * @param dataSchema + * @returns properties Record[] + */ + flattenSchema = (dataSchema: Record, type: string): Record[] => { + + const properties: Record[] = [] + const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { + _.mapKeys(schema, function (value, parentKey) { + const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; + const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); + switch (value["type"]) { + case "object": + flatten(_.get(value, "properties"), newKey, newExpr); + break; + case "array": + if (type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { + _.mapKeys(_.get(value, "items.properties"), function (value, childKey) { + const objChildKey = _.join([newKey, childKey], ".") + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), { expr: _.join([newExpr, "[*].['", childKey, "']"], ""), name: objChildKey, data_type: "array" })) + }) + } else { + properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), { expr: newExpr + "[*]", name: newKey, type: _.get(value, "items.type") })) + } + break; + default: + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), { expr: newExpr, name: newKey })) + } + }); + } + flatten(_.get(dataSchema, "properties"), undefined, undefined) + return properties + } + + /** + * Get all fields of a dataset merging schema fields, transformations and denorm fields + * + * @param data_schema + * @param transformations_config + * @param denorm_config + * @returns Promise[]> + */ + getAllFields = async (dataset: Record, type: string): Promise[]> => { + + const { data_schema, denorm_config, transformations_config } = dataset + let dataFields = this.flattenSchema(data_schema, type); + if (!_.isEmpty(denorm_config.denorm_fields)) { + for (const denormField of denorm_config.denorm_fields) { + const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); + const properties = this.flattenSchema(denormDataset.data_schema, type); + const transformProps = _.map(properties, (prop) => { + _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + return prop; + }); + dataFields.push(...transformProps); + } + } + if (!_.isEmpty(transformations_config)) { + const transformationFields = _.map(transformations_config, (tf) => ({ + expr: "$." + tf.field_key, + name: tf.field_key, + data_type: tf.transformation_function.datatype, + arrival_format: tf.transformation_function.datatype, + type: tf.transformation_function.datatype + })) + const originalFields = _.differenceBy(dataFields, transformationFields, "name") + dataFields = _.concat(originalFields, transformationFields) + } + dataFields.push(rawIngestionSpecDefaults.synctsField) + _.remove(dataFields, { is_deleted: true }) // Delete all the excluded fields + return dataFields; + } +} + +class TableGenerator extends BaseTableGenerator { + + getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { + + const { dataset_config, router_config } = dataset + return { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": datasourceRef, + "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset), dataset_config.keys_config.partition_key) }, + "timestampSpec": { "column": this.getTimestampKey(dataset), "format": "auto" }, + "metricsSpec": [], + "granularitySpec": rawIngestionSpecDefaults.granularitySpec + }, + "tuningConfig": rawIngestionSpecDefaults.tuningConfig, + "ioConfig": _.merge(rawIngestionSpecDefaults.ioConfig, { + "topic": router_config.topic, + "inputFormat": { + "flattenSpec": { + "fields": this.getDruidFlattenSpec(allFields) + } + } + }) + } + } + } + + private getDruidDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { + + const dataFields = _.cloneDeep(allFields); + if (partitionKey) { // Move the partition column to the top of the dimensions + const partitionCol = _.remove(dataFields, { name: partitionKey }) + if (partitionCol && _.size(partitionCol) > 0) { + dataFields.unshift(partitionCol[0]) + } + } + _.remove(dataFields, { name: timestampKey }) + return _.union( + _.map(dataFields, (field) => { + return { + "type": this.getDruidDimensionType(field.data_type), + "name": field.name + } + }), + rawIngestionSpecDefaults.dimensions + ) + } + + private getDruidDimensionType = (data_type: string): string => { + switch (data_type) { + case "number": return "double"; + case "integer": return "long"; + case "object": return "json"; + case "boolean": return "string"; + case "array": return "json"; + case "string": return "string"; + default: return "auto"; + } + } + + private getDruidFlattenSpec = (allFields: Record) => { + const allfields = _.map(allFields, (field) => { + return { + type: "path", + expr: field.expr, + name: field.name + } + }); + return _.uniqBy([...allfields, ...rawIngestionSpecDefaults.flattenSpec], "name") + } + + getHudiIngestionSpecForCreate = (dataset: Record, allFields: Record[], datasourceRef: string) => { + + const primaryKey = this.getPrimaryKey(dataset); + const partitionKey = this.getHudiPartitionKey(dataset); + const timestampKey = this.getTimestampKey(dataset); + return { + dataset: dataset.dataset_id, + schema: { + table: datasourceRef, + partitionColumn: partitionKey, + timestampColumn: timestampKey, + primaryKey: primaryKey, + columnSpec: this.getHudiColumnSpec(allFields, primaryKey, partitionKey, timestampKey) + }, + inputFormat: { + type: "json", + flattenSpec: { + fields: this.getHudiFields(allFields) + } + } + } + } + + getHudiIngestionSpecForUpdate = (dataset: Record, existingHudiSpec: Record, allFields: Record[], datasourceRef: string) => { + + const newHudiSpec = this.getHudiIngestionSpecForCreate(dataset, allFields, datasourceRef) + + const newColumnSpec = newHudiSpec.schema.columnSpec; + const oldColumnSpec = existingHudiSpec.schema.columnSpec; + let currIndex = _.get(_.maxBy(oldColumnSpec, "index"), "index") as unknown as number + const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, "name"); + if (_.size(newColumns) > 0) { + _.each(newColumns, (col) => { + oldColumnSpec.push({ + "type": col.type, + "name": col.name, + "index": currIndex++ + }) + }) + } + _.set(newHudiSpec, "schema.columnSpec", oldColumnSpec) + return newHudiSpec; + } + + private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { + + const dataFields = _.cloneDeep(allFields); + _.remove(dataFields, { name: primaryKey }) + _.remove(dataFields, { name: partitionKey }) + _.remove(dataFields, { name: timestampKey }) + let index = 1; + const transformFields = _.map(dataFields, (field) => { + return { + "type": this.getHudiColumnType(field), + "name": field.name, + "index": index++ + } + }) + _.each(rawIngestionSpecDefaults.dimensions, (field) => { + transformFields.push({ + "type": field.type, + "name": field.name, + "index": index++ + }) + }) + return transformFields; + } + + private getHudiColumnType = (field: Record): string => { + if (field.data_type === "array" && field.arrival_format !== "array") { + return "array"; + } + if (field.data_type === "array" && field.arrival_format === "array") { + switch (field.type) { + case "string": + return "array" + case "number": + return "array" + case "integer": + return "array" + case "boolean": + return "array" + default: + return "array" + } + } + switch (field.arrival_format) { + case "text": + return "string" + case "number": + switch (field.data_type) { + case "integer": + return "int" + case "epoch": + return "epoch" + case "bigdecimal": + return "bigdecimal" + case "float": + return "float" + case "long": + return "long" + default: + return "double" + } + case "integer": + return "int" + case "boolean": + return "boolean" + default: + return "string" + } + } + + private getHudiFields = (allFields: Record[]): Record[] => { + + const regexString = "[\\[\\]'\\*]"; + const regex = new RegExp(regexString, "g"); + return _.union( + _.map(allFields, (field) => { + return { + type: "path", + expr: _.replace(field.expr, regex, ""), + name: field.name + } + }), + rawIngestionSpecDefaults.flattenSpec + ) + } + + private getPrimaryKey = (dataset: Record): string => { + return dataset.dataset_config.keys_config.data_key; + } + + private getHudiPartitionKey = (dataset: Record): string => { + return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; + } + + private getTimestampKey = (dataset: Record): string => { + return dataset.dataset_config.keys_config.timestamp_key; + } +} + +export const tableGenerator = new TableGenerator(); \ No newline at end of file diff --git a/api-service/src/v2/services/ValidationService.ts b/api-service/src/services/ValidationService.ts similarity index 100% rename from api-service/src/v2/services/ValidationService.ts rename to api-service/src/services/ValidationService.ts diff --git a/api-service/src/v1/services/WrapperService.ts b/api-service/src/services/WrapperService.ts similarity index 98% rename from api-service/src/v1/services/WrapperService.ts rename to api-service/src/services/WrapperService.ts index 37743d5f..c38afb6d 100644 --- a/api-service/src/v1/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -1,11 +1,11 @@ import axios from "axios"; import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; import { config } from "../configs/Config"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -export class WrapperService { +class WrapperService { + private errorHandler: ErrorResponseHandler; constructor() { this.errorHandler = new ErrorResponseHandler("WrapperService"); @@ -104,3 +104,5 @@ export class WrapperService { } } + +export const wrapperService = new WrapperService() \ No newline at end of file diff --git a/api-service/src/services/fs.ts b/api-service/src/services/fs.ts new file mode 100644 index 00000000..cfc0010b --- /dev/null +++ b/api-service/src/services/fs.ts @@ -0,0 +1,18 @@ +import fs from "fs"; +import path from "path"; + +export const scrapModules = (folderPath: string, basename: string) => { + const mapping = new Map>(); + fs.readdirSync(folderPath) + .filter((file) => file !== basename) + .map((file) => { + const { + default: { name, ...others }, + /* eslint-disable @typescript-eslint/no-var-requires */ + } = require(path.join(folderPath, file)) as { default: Type }; + + mapping.set(name, others); + }); + + return mapping; +}; diff --git a/api-service/src/services/managers/constants.ts b/api-service/src/services/managers/constants.ts new file mode 100644 index 00000000..d6d9c028 --- /dev/null +++ b/api-service/src/services/managers/constants.ts @@ -0,0 +1,32 @@ +export default { + "RECORD_ALREADY_PUBLISHED": "Record is already published", + "RECORD_NOT_FOUND": "Record does not exist in the system", + "PROMETHEUS_DATASOURCE_NOT_FOUND": "Prometheus data source does not exist", + "RULE_ALREADY_EXIST": "Alert Rule already exists", + "RULE_ALREADY_PUBLISHED": "Alert Rule already published", + "INVALID_MANAGER": "Invalid Alert manager", + "UPDATE_FAILURE": "Failed to update record", + "INVALID_QUERY": "Invalid Query", + "RECORD_ALREADY_DELETED": "Record is already deleted", + "FOLDER_NOT_FOUND": "Alert folder not found", + "CATEGORY_NOT_EXIST": "Category does not exists", + "UPDATE_STATUS_FAILURE": "Failed to update alert status", + "WEBHOOK_MISSING": "webhook url is missing", + "INTERNAL_SERVER_ERROR": "Internal Server Error", + "DEFAULT_TEST_NOTIFICATION_MESSAGE": "Hello World", + "METHOD_NOT_IMPLEMENTED": "method not implemented", + "SILENCE_EXPIRED": "Silence is already expired", + "ALERTS_NOT_RETIRED": "Failed to retire alerts for dataset", + "ALERTS_FETCH_FAILURE": "Failed to fetch alerts for dataset", + "ALERTS_RETIRED_SUCCESSFULLY": "Alerts retired successfully for datasets", + "SILENCE_DELETED_SUCCESSFULLY":"Silence deleted successfully for the Alert rule", + "ALERTS_PUBLISHED_SUCCESSFULLY": "Alerts published successfully for datasets", + "ALERT_PUBLISH_FAILURE":"Failed to publish alerts for dataset", + "ALERT_CREATE_FAILURE":"Failed to create alerts for datasets", + "METRIC_ALIAS_CREATE_FAILURE":"Failed to create metric alias for datasets", + "ALERTS_NOT_FOUND":"Alert rule does not exist", + "METRIC_ALIAS_NOT_FOUND":"Metric alias does not exist", + "METRIC_ALIAS_FETCH_FAILURE":"Failed to fetch alert metrics for dataset", + "METRIC_ALIAS_NOT_DELETED": "Failed to delete matric alias for datasets", + "METRIC_ALIAS_DELETED_SUCCESSFULLY":"Metric alias deleted successfully for datasets" +} diff --git a/api-service/src/services/managers/grafana/alert/helpers/index.ts b/api-service/src/services/managers/grafana/alert/helpers/index.ts new file mode 100644 index 00000000..9c577a52 --- /dev/null +++ b/api-service/src/services/managers/grafana/alert/helpers/index.ts @@ -0,0 +1,302 @@ +import _ from "lodash"; + +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; +import constants from "../../../constants"; +import { Notification } from "../../../../../models/Notification"; + +export const getRules = () => { + return grafanaHttpClient.get("/api/ruler/grafana/api/v1/rules"); +}; + +const getDatasource = () => { + return grafanaHttpClient.get("api/datasources"); +}; + +const alerts = () => { + return grafanaHttpClient.get("/api/prometheus/grafana/api/v1/rules"); +}; + +const deleteAlertRule = (alertCategory: string) => { + return grafanaHttpClient.delete(`/api/ruler/grafana/api/v1/rules/${alertCategory}/${alertCategory}`); +}; + +const getFilteredAlerts = () => { + return grafanaHttpClient.get("/api/alertmanager/grafana/api/v2/alerts?silenced&active&inhibited"); +} + +const deleteAlertFolder = async (folderName: string) => { + const folderUID = await getFolderUid(folderName); + if (!folderUID) throw new Error(constants.FOLDER_NOT_FOUND); + return grafanaHttpClient.delete(`/api/folders/${folderUID}`) +}; + +const checkIfGroupNameExists = async (category: string) => { + const response = await getRules(); + const rules = _.get(response, "data"); + if(!_.has(rules, category)) return undefined; + return _.find(_.flatMap(_.values(rules)), { + name: category, + }); +}; + +const checkIfRuleExists = (category: string | any, ruleName: string) => { + const ruleExists = category.rules.some((rule: any) => rule.grafana_alert.title === ruleName); + if (ruleExists) { + throw new Error(constants.RULE_ALREADY_EXIST); + } +}; + +const getPrometheusDataSource = async () => { + const dataSources = await getDatasource(); + const prometheusDataSource = _.find(dataSources.data, { type: "prometheus" }); + if (!prometheusDataSource) { + throw new Error(constants.PROMETHEUS_DATASOURCE_NOT_FOUND); + } + return prometheusDataSource; +}; + +const getSpecificRule = async (payload: Record) => { + const alertrules = await alerts(); + const groups = _.get(alertrules, "data.data.groups"); + const ruleGroup = _.find(groups, (group: any) => group.name == payload.category); + return _.find(_.get(ruleGroup, "rules"), (rule: any) => rule.name == payload.name); +}; + +const updateMetadata = (metadata: any, dataSource: string, expression: string) => { + const str = JSON.stringify(metadata); + metadata = _.replace(_.replace(str, /\$datasourceUid/g, dataSource), /\$expr/g, expression); + metadata = JSON.parse(metadata); + return metadata; +}; + +const addGrafanaRule = (payload: Record) => { + return grafanaHttpClient.post(`/api/ruler/grafana/api/v1/rules/${payload.name}`, payload); +}; + +const getFolders = () => { + return grafanaHttpClient.get("/api/folders") +}; + +const getFolderUid = async (name: string) => { + const folders = await getFolders() + const folderPayload = _.find(folders.data, (folder: any) => folder.title == name) || []; + return folderPayload.uid; +} +const createFolder = (title: string) => { + const payload = { title }; + return grafanaHttpClient.post("/api/folders", payload); +}; + +const createFolderIfNotExists = async (folderName: string) => { + const folders = await getFolders(); + const isExists = _.find(folders.data, folder => _.get(folder, "title") === folderName); + if (isExists) return; + return createFolder(folderName); +} + +const getQueryModel = (payload: Record) => { + const { metric, operator, threshold } = payload; + const relativeTimeRange = { "from": 600, "to": 0 }; + + return [ + { + "refId": "A", + "datasourceUid": "$datasourceUid", + "queryType": "", + "relativeTimeRange": relativeTimeRange, + "model": { + "refId": "A", + "hide": false, + "editorMode": "code", + "expr": metric, + "legendFormat": "__auto", + "range": true + } + }, + { + "refId": "B", + "datasourceUid": "__expr__", + "queryType": "", + "model": { + "refId": "B", + "hide": false, + "type": "reduce", + "datasource": { + "uid": "__expr__", + "type": "__expr__" + }, + "settings": { + "mode": "replaceNN", + "replaceWithValue": 0 + }, + "conditions": [ + { + "type": "query", + "evaluator": { + "params": [], + "type": operator + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B" + ] + }, + "reducer": { + "params": [], + "type": "last" + } + } + ], + "reducer": "last", + "expression": "A" + }, + "relativeTimeRange": relativeTimeRange + }, + { + "refId": "C", + "datasourceUid": "__expr__", + "queryType": "", + "model": { + "refId": "C", + "hide": false, + "type": "threshold", + "datasource": { + "uid": "__expr__", + "type": "__expr__" + }, + "conditions": [ + { + "type": "query", + "evaluator": { + "params": threshold, + "type": operator + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + } + } + ], + "expression": "B" + }, + "relativeTimeRange": relativeTimeRange + } + ] +} + +const queryOperators = [ + { + label: "greater than (>)", + value: "gt", + symbol: ">" + }, + { + label: "less than (<)", + value: "lt", + symbol: "<" + }, + { + "label": "within range", + "value": "within_range", + "symbol": "within_range" + } +]; + +const getQueryExpression = (payload: Record) => { + const { metric, operator, threshold } = payload; + const operatorSymbol = _.get(_.find(queryOperators, operatorMetadata => _.get(operatorMetadata, "value") === operator), "symbol"); + return `(${metric}) ${operatorSymbol} ${threshold}`; +} + +const getMatchingLabels = async (channels: string[]) => { + try { + + const fetchChannel = (id: string) => { + return Notification.findOne({ where: { id } }) + .then(response => response?.toJSON()) + .then(channelMetadata => { + const { name, type } = channelMetadata; + return `notificationChannel_${name}_${type}`; + }) + .catch(() => null); + } + + const matchingLabels = await Promise.all(channels.map(fetchChannel)); + return _.reduce(_.compact(matchingLabels), (acc, value) => { + return { ...acc, [value]: "true" }; + }, {}) + + } catch (error) { + return {} + } +} + +const transformRule = async ({ value, condition, metadata, isGroup }: any) => { + const { name, id, interval, category, frequency, labels = {}, annotations = {}, severity, description, notification = {} } = value; + const annotationObj = { ...annotations, description: description }; + const channels = _.get(notification, "channels") || []; + const matchingLabelsForNotification = await getMatchingLabels(channels); + + const payload = { + grafana_alert: { + title: name, + condition: condition, + no_data_state: _.get(metadata, "no_data_state", "NoData"), + exec_err_state: _.get(metadata, "exec_err_state", "Error"), + data: metadata, + is_paused: false, + }, + for: interval, + annotations: annotationObj, + labels: { + "alertId": id, + ...labels, + ...(severity && { severity }), + ...matchingLabelsForNotification + } + }; + + if (isGroup) { + return { name: category, interval: frequency, rules: [payload] }; + } + + return payload; +}; + + +const groupRulesByCategory = (payload: Record[]) => { + return _.reduce(payload, (accumulator: Record, current: Record) => { + const { category, name } = current; + const existing = _.get(accumulator, category) || []; + accumulator[category] = [...existing, name]; + return accumulator + }, {}); +} + +export { + getPrometheusDataSource, + updateMetadata, + checkIfGroupNameExists, + checkIfRuleExists, + transformRule, + createFolderIfNotExists, + addGrafanaRule, + getSpecificRule, + deleteAlertRule, + deleteAlertFolder, + getQueryExpression, + getQueryModel, + getFilteredAlerts, + groupRulesByCategory +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/alert/index.ts b/api-service/src/services/managers/grafana/alert/index.ts new file mode 100644 index 00000000..471a104d --- /dev/null +++ b/api-service/src/services/managers/grafana/alert/index.ts @@ -0,0 +1,114 @@ +import _ from "lodash"; +import { addGrafanaRule, checkIfGroupNameExists, checkIfRuleExists, createFolderIfNotExists, deleteAlertFolder, deleteAlertRule, getPrometheusDataSource, getQueryExpression, getQueryModel, getSpecificRule, transformRule, updateMetadata, getRules } from "./helpers"; +import { Silence } from "../../../../models/Silence"; +import constants from "../../constants"; + +const publishAlert = async (payload: Record) => { + let metaData = payload.metadata || []; + const conditionRef: any = _.last(metaData.query); + if (!conditionRef) throw new Error(constants.INVALID_QUERY); + const prometheusDataSource = await getPrometheusDataSource(); + metaData = await updateMetadata(metaData, _.get(prometheusDataSource, "uid"), payload.expression); + const categoryExists = await checkIfGroupNameExists(payload.category); + let alertRulePayload; + const transformPayload = { value: payload, condition: conditionRef.refId, metadata: metaData.query }; + if (categoryExists) { + checkIfRuleExists(categoryExists, payload.name); + const newRule = await transformRule({ ...transformPayload, isGroup: false }); + const transformRules = _.concat(categoryExists.rules, newRule); + alertRulePayload = { ...categoryExists, rules: transformRules }; + } else { + await createFolderIfNotExists(payload.category); + alertRulePayload = await transformRule({ ...transformPayload, isGroup: true }); + } + + return addGrafanaRule(alertRulePayload); +}; + +const getAlerts = async (payload: Record) => { + const context = payload?.context || {}; + const alertId = _.get(payload, "id"); + const { err, alertData } = await getSpecificRule(payload) + .then(alertData => { + if (!alertData) throw new Error() + return { err: null, alertData } + }) + .catch(err => ({ err: err?.message || "rule not found in alerting manager", alertData: null })); + + const silenceModel = await Silence.findOne({ where: { alert_id: alertId } }); + const silenceData = silenceModel?.toJSON(); + const silenceState: Record = { state: "", silenceId: "" }; + + if (silenceData) { + const { end_time } = silenceData; + const currentTime = new Date().getTime(); + const endTime = new Date(end_time).getTime(); + if (currentTime < endTime) { + silenceState.state = "muted"; + silenceState["endTime"] = endTime; + } else { + silenceState.state = "unmuted"; + } + silenceState.silenceId = silenceData.id; + } else { + silenceState.state = "unmuted"; + } + + return { ...payload, context: { ...context, err }, ...(alertData && { alertData }), silenceState }; +}; + +const deleteAlert = async (payload: Record) => { + const { name, category } = payload; + const alertCategory = await checkIfGroupNameExists(category); + if (!alertCategory) throw new Error(constants.CATEGORY_NOT_EXIST); + + if (_.get(alertCategory, "rules.length") > 1) { + const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, "grafana_alert.title") !== name) || []; + const filteredGroup = { ...alertCategory, rules: filteredRule }; + return addGrafanaRule(filteredGroup); + } + + await deleteAlertRule(category); + return deleteAlertFolder(category); +} + +const generateAlertPayload = (payload: Record) => { + if (!(_.get(payload, "metadata.queryBuilderContext"))) return payload; + const { metadata } = payload; + const { queryBuilderContext } = metadata; + const expression = getQueryExpression(queryBuilderContext); + const queryModel = getQueryModel(queryBuilderContext); + const updatedMetadata = { ...metadata, query: queryModel } + return { ...payload, expression, metadata: updatedMetadata } +} + +const filterSystemRulesPredicate = (rule: Record) => { + const labels = _.get(rule, "labels") || {}; + const isSystemAlert = _.find(labels, (value, key) => (key === "alertSource" && value === "system-rule-ingestor-job")); + if (!isSystemAlert) return true; + return false; +} + +const deleteSystemRules = async () => { + const response = await getRules(); + const existingRules = _.cloneDeep(_.get(response, "data")) as Record[]>; + for (const [category, evaluationGroups] of Object.entries(existingRules)) { + const evaluationGroup = _.find(evaluationGroups, ["name", category]); + if (!evaluationGroup) continue; + const rules = _.get(evaluationGroup, "rules") || [] + const filteredRules = _.filter(rules, filterSystemRulesPredicate); + try { + if (_.isEmpty(filteredRules)) { + await deleteAlertRule(category); + await deleteAlertFolder(category); + } else { + const filteredGroup = { ...evaluationGroup, rules: filteredRules }; + await addGrafanaRule(filteredGroup); + } + } catch (err) { + console.log(err) + } + } +} + +export { publishAlert, generateAlertPayload, deleteAlert, getAlerts, deleteSystemRules } \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/index.ts b/api-service/src/services/managers/grafana/index.ts new file mode 100644 index 00000000..1824c6c5 --- /dev/null +++ b/api-service/src/services/managers/grafana/index.ts @@ -0,0 +1,6 @@ +import * as alertFunctions from "./alert" +import * as notificationFunctions from "./notification"; +import * as silenceFunctions from "./silences"; + +const service = { name: "grafana", ...alertFunctions, ...notificationFunctions, ...silenceFunctions }; +export default service \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/email.ts b/api-service/src/services/managers/grafana/notification/channels/email.ts new file mode 100644 index 00000000..06658016 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/email.ts @@ -0,0 +1,53 @@ +import _ from "lodash"; +import { IChannelConfig } from "../../../../../types/AlertModels"; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; + +const getReceiverObject = ({ name, recipientAddresses, message, multipleAddresses, type }: any) => { + return { + name, + grafana_managed_receiver_configs: [ + { + settings: { + addresses: recipientAddresses, + subject: "{{template \"email\" .}}", + singleEmail: !multipleAddresses, + ...(message && { + message + }) + }, + secureSettings: {}, + type: type, + name, + disableResolveMessage: false + } + ] + } +} + +const service: IChannelConfig = { + name: "email", + service: { + generateConfigPayload(payload: Record): Record { + const { type, config, name } = payload; + const { recipientAddresses, message, subject = "Obsrv Alert", labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; + const multipleAddresses = _.size(_.split(recipientAddresses, ";")) > 1; + return { + notificationPolicy: { + receiver: name, + object_matchers: labels + }, + receiver: getReceiverObject({ name, type, message, multipleAddresses, recipientAddresses, subject }) + } + }, + testChannel(payload: Record): Promise { + const { name, type, config, message = "Test Channel" } = payload; + const { recipientAddresses, subject = "Obsrv Alert" } = config; + const multipleAddresses = _.size(_.split(recipientAddresses, ";")) > 1; + const alert = { annotations: { description: message }, labels: {} }; + const body = { alert, receivers: [getReceiverObject({ name, type, multipleAddresses, recipientAddresses, subject })] }; + return grafanaHttpClient.post("api/alertmanager/grafana/config/api/v1/receivers/test", body); + } + } +} + +export default service; \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/index.ts b/api-service/src/services/managers/grafana/notification/channels/index.ts new file mode 100644 index 00000000..abc6b45a --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/index.ts @@ -0,0 +1,11 @@ +import path from "path"; +import { scrapModules } from "../../../../../services/fs"; +import { IChannelConfig } from "../../../../../types/AlertModels"; + +const channels = scrapModules(__dirname, path.basename(__filename)); + +export const getChannelService = (channelName: string) => { + const channel = channels.get(channelName.toLowerCase()); + if (!channel) throw new Error("invalid channel"); + return channel; +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/slack.ts b/api-service/src/services/managers/grafana/notification/channels/slack.ts new file mode 100644 index 00000000..983d9879 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/slack.ts @@ -0,0 +1,50 @@ +import axios from "axios"; +import CONSTANTS from "../../../constants" +import { IChannelConfig } from "../../../../../types/AlertModels"; + +const generateConfigPayload = (payload: Record): Record => { + const { type, config, name } = payload; + const { webhookUrl, labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; + + return { + notificationPolicy: { + receiver: name, + object_matchers: labels + }, + receiver: { + name, + grafana_managed_receiver_configs: [ + { + settings: { + title: "{{ template \"slack_title\" . }}", + text: "{{ template \"slack_body\" . }}" + }, + secureSettings: { + token: "", + url: webhookUrl + }, + type, + name, + disableResolveMessage: false + } + ] + } + } +} + +const testChannel = (payload: Record): Promise => { + const { config, message } = payload; + const { webhookUrl } = config; + if (!webhookUrl) throw new Error(CONSTANTS.WEBHOOK_MISSING); + return axios.post(webhookUrl, { text: message }, { headers: { "Content-Type": "application/json" } }) +} + +const service: IChannelConfig = { + name: "slack", + service: { + generateConfigPayload, + testChannel + } +} + +export default service \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/teams.ts b/api-service/src/services/managers/grafana/notification/channels/teams.ts new file mode 100644 index 00000000..714f4f19 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/teams.ts @@ -0,0 +1,47 @@ +import axios from "axios"; +import CONSTANTS from "../../../constants"; +import { IChannelConfig } from "../../../../../types/AlertModels"; + +const generateConfigPayload = (payload: Record): Record => { + const {type, config, name} = payload; + const { webhookUrl, labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; + return { + notificationPolicy: { + receiver: name, + object_matchers: labels, + }, + receiver: { + name, + grafana_managed_receiver_configs: [ + { + settings: { + url: webhookUrl, + title: "{{template \"teams.alerts.title\" .}}", + text: "{{template \"teams.alerts.message\" .}}" + }, + secureSettings: {}, + name, + type, + disableResolveMessage: false, + } + ] + } + } +} + +const testChannel = (payload: Record): Promise => { + const {config, message} = payload; + const {webhookUrl} = config; + if(!webhookUrl) throw new Error(CONSTANTS.WEBHOOK_MISSING) + return axios.post(webhookUrl, {text: message}, {headers: {"Content-Type": "application/json"}}) +} + +const service: IChannelConfig = { + name: "teams", + service: { + generateConfigPayload, + testChannel + } +} + +export default service; \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/helpers/index.ts b/api-service/src/services/managers/grafana/notification/helpers/index.ts new file mode 100644 index 00000000..112365b7 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/helpers/index.ts @@ -0,0 +1,56 @@ +import _ from "lodash"; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; +import { getChannelService } from "../channels"; +import defaultTemplates from "../templates"; + +const generateChannelConfig = (payload: Record) => { + const { type } = payload; + const channel = getChannelService(type); + return channel.service.generateConfigPayload(payload); +}; + +const getAlertManagerConfig = async () => { + return grafanaHttpClient.get("/api/alertmanager/grafana/config/api/v1/alerts") + .then(response => response.data); +}; + +const updateAlertManagerConfig = async (payload: Record) => { + return grafanaHttpClient.post("/api/alertmanager/grafana/config/api/v1/alerts", payload); +}; + +const getReceivers = (alertmanager_config: Record) => _.get(alertmanager_config, "alertmanager_config.receivers") as Array; +const getRoutes = (alertmanager_config: Record) => _.get(alertmanager_config, "alertmanager_config.route.routes") as Array; +const getTemplates = (alertmanager_config: Record) => _.get(alertmanager_config, "template_files") as Record; + +const createContactPointsAndNotificationPolicy = async (metadata: Record) => { + const { receiver, notificationPolicy } = metadata; + const config = await getAlertManagerConfig(); + const existingReceivers = getReceivers(config) || []; + const existingRoutes = getRoutes(config) || []; + const existingTemplates = getTemplates(config) || {}; + _.set(config, "alertmanager_config.receivers", [...existingReceivers, receiver]); + _.set(config, "alertmanager_config.route.routes", [...existingRoutes, notificationPolicy]); + _.set(config, "template_files", { ...existingTemplates, ...defaultTemplates }); + return updateAlertManagerConfig(config); +}; + +const removeReceiverAndNotificationPolicy = (payload: Record, alertManagerConfig: Record) => { + const { name } = payload; + const clonedAlertManagerConfig = _.cloneDeep(alertManagerConfig); + const existingReceivers = getReceivers(clonedAlertManagerConfig) || []; + const existingRoutes = getRoutes(clonedAlertManagerConfig) || []; + _.remove(existingRoutes, route => _.get(route, "receiver") === name); + _.remove(existingReceivers, receiver => _.get(receiver, "name") === name); + return clonedAlertManagerConfig; +}; + +export { + generateChannelConfig, + getAlertManagerConfig, + updateAlertManagerConfig, + getReceivers, + getRoutes, + getTemplates, + createContactPointsAndNotificationPolicy, + removeReceiverAndNotificationPolicy +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/index.ts b/api-service/src/services/managers/grafana/notification/index.ts new file mode 100644 index 00000000..816e473c --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/index.ts @@ -0,0 +1,26 @@ +import { getChannelService } from "./channels"; +import { createContactPointsAndNotificationPolicy, generateChannelConfig, getAlertManagerConfig, removeReceiverAndNotificationPolicy, updateAlertManagerConfig } from "./helpers"; + +const updateNotificationChannel = async (payload: Record) => { + const alertManagerConfig = await getAlertManagerConfig(); + const updatedAlertManagerConfig = removeReceiverAndNotificationPolicy(payload, alertManagerConfig); + return updateAlertManagerConfig(updatedAlertManagerConfig); +}; + +const createNotificationChannel = (payload: Record) => { + const { type, config, name } = payload; + const { notificationPolicy, receiver } = generateChannelConfig({ type, config, name }); + return createContactPointsAndNotificationPolicy({ notificationPolicy, receiver }); +}; + +const testNotificationChannel = async (payload: Record) => { + const { type } = payload; + const channel = getChannelService(type); + return channel.service.testChannel(payload); +}; + +export { + updateNotificationChannel, + createNotificationChannel, + testNotificationChannel +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/templates/index.ts b/api-service/src/services/managers/grafana/notification/templates/index.ts new file mode 100644 index 00000000..cf00f1c0 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/templates/index.ts @@ -0,0 +1,9 @@ +const templates = { + "email": "{{ define \"email\" }}\n{{ range .Alerts.Firing }}\nObsrv - {{.Labels.alertname}}\n{{ end }}\n{{ end }}", + "slack_title": "{{ define \"slack_title\" }}\n[{{ .Status | toUpper }}: {{ if eq .Status \"firing\" }}{{ .Alerts.Firing | len }}{{ else if eq .Status \"resolved\" }}{{.Alerts.Resolved | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join \" \" }}\n{{ end }}", + "slack_body": "{{ define \"slack_body\" -}}\n{{- $status := \"resolved\" -}}\n{{ range $index, $alert := .Alerts }}\n{{ if eq $index 0 }}\n Severity: {{ .Labels.severity }}\n Description: {{ .Annotations.description }}\n Status: {{ .Status }}\n{{ end }}\n{{ end }}\n{{ end }}", + "teams.alerts.title": "{{ define \"teams.alerts.title\" }}\n{{ .CommonLabels.alertname }}\n{{ end }}", + "teams.alerts.message": "{{ define \"teams.alerts.message\" }}\n\n{{ if gt (len .Alerts.Firing) 0 }} **Firing**\n{{ template \"__teams_text_alert_list\" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}\n\n{{ end }}\n{{ end }}\n{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**\n{{ template \"__teams_text_alert_list\" .Alerts.Resolved }}{{ end }}{{ end }}\n\n{{ define \"__teams_text_alert_list\" }}\n{{ range . }}\n\n{{ if gt (len .Labels.severity) 0 }}\nSeverity = {{ .Labels.severity }}\n{{ end }}\nDescription = {{ .Annotations.description }}\n{{ end }}\n{{ end }}" +} + +export default templates; \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/silences/helpers/index.ts b/api-service/src/services/managers/grafana/silences/helpers/index.ts new file mode 100644 index 00000000..50af380c --- /dev/null +++ b/api-service/src/services/managers/grafana/silences/helpers/index.ts @@ -0,0 +1,50 @@ +import _ from "lodash"; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; + +const addSilence = (payload: Record) => { + return grafanaHttpClient.post("/api/alertmanager/grafana/api/v2/silences", payload); +} + +const fetchAllSilences = () => { + return grafanaHttpClient.get("/api/alertmanager/grafana/api/v2/silences"); +} + +const getSilence = (silenceId: string) => { + return grafanaHttpClient.get(`/api/alertmanager/grafana/api/v2/silence/${silenceId}`); +} + +const disableSilence = (silenceId: string) => { + return grafanaHttpClient.delete(`/api/alertmanager/grafana/api/v2/silence/${silenceId}`); +} + +const getCurrentSilenceStatus = async (silenceId: string) => { + const response = await getSilence(silenceId); + const currentSilenceStatus = _.get(response, "data.status.state"); + return currentSilenceStatus; +} + +const transfromPayload = (alertId: string, startDate: string, endDate: string, silenceId?: string,) => { + return { + "comment": "Silence for alert: " + alertId, + "createdby": "admin", + "startsat": startDate, + "endsat": endDate, + "id": silenceId, + "matchers": [ + { + name: "alertId", + value: alertId, + isRegex: false, + isEqual: true + } + ] + } +} + +export { + addSilence, + fetchAllSilences, + disableSilence, + getCurrentSilenceStatus, + transfromPayload +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/silences/index.ts b/api-service/src/services/managers/grafana/silences/index.ts new file mode 100644 index 00000000..95e23b21 --- /dev/null +++ b/api-service/src/services/managers/grafana/silences/index.ts @@ -0,0 +1,43 @@ +import _ from "lodash"; +import { addSilence, disableSilence, getCurrentSilenceStatus, transfromPayload } from "./helpers"; + +const createSilence = async (payload: Record) => { + const { alertId, startDate, endDate, manager } = payload; + const grafanaPayload = transfromPayload(alertId, startDate, endDate); + + try { + const addSilenceResponse = await addSilence(grafanaPayload); + const silenceId = addSilenceResponse.data.silenceID; + + return { + silenceId, + manager, + } + + } catch (err: any) { + throw new Error(err); + } +} + +const getSilenceMetadata = async (payload: Record) => { + const silenceId = _.get(payload, "id") + const silenceStatus = await getCurrentSilenceStatus(silenceId); + return { + ...payload, + status: silenceStatus + } +} + +const updateSilence = async (silence: Record, payload: Record) => { + const { id, alert_id } = silence; + const { startTime, endTime } = payload; + const grafanaPayload = transfromPayload(alert_id, startTime, endTime, id); + await addSilence(grafanaPayload); +} + + +const deleteSilence = (silenceId: string) => { + return disableSilence(silenceId); +} + +export { createSilence, getSilenceMetadata, updateSilence, deleteSilence } \ No newline at end of file diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts new file mode 100644 index 00000000..84b666d9 --- /dev/null +++ b/api-service/src/services/managers/index.ts @@ -0,0 +1,217 @@ +import _ from "lodash"; +import { Alert } from "../../models/Alert"; +import grafanaService from "./grafana/index"; +import prometheusService from "./prometheus/index"; +import { Silence } from "../../models/Silence"; +import { Metrics } from "../../models/Metric"; +import constants from "./constants"; + +export const getAlertRule = (id: string) => { + return Alert.findOne({ where: { id } }); +} + +const getService = (manager: string) => { + switch (manager) { + case "grafana": { + return grafanaService; + } + case "prometheus": { + return prometheusService; + } + default: + throw new Error("Invalid Alert manager"); + } +}; + +export const publishAlert = async (payload: Record) => { + const { id, manager } = payload; + const service = getService(manager); + const publishResponse = await service.publishAlert(payload) + await updateStatus(id, "live"); + return publishResponse; +}; + + +const updateStatus = (id: string, status: string) => { + return Alert.update({ status }, { where: { id } }); +} + +const deleteRule = (id: string) => { + return Alert.destroy({ where: { id } }) +} + +export const deleteAlertRule = async (payload: Record, hardDelete: boolean) => { + const { id, manager, status } = payload; + + if (status == "live") { + try { + const service = getService(manager); + await service.deleteAlert(payload) + } catch (err: any) { + console.log(err) + } + } + + if (hardDelete) { + return deleteRule(id); + } + + return updateStatus(id, "retired"); +} + + +export const deleteSystemRules = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.deleteSystemRules(); +} + +export const getAlertsMetadata = (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.getAlerts(payload); +} + +export const getAlertPayload = (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.generateAlertPayload(payload); +} + +export const publishNotificationChannel = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.createNotificationChannel(payload); +} + +export const testNotificationChannel = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.testNotificationChannel(payload); +} + +export const updateNotificationChannel = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.updateNotificationChannel(payload); +} + +export const createSilence = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.createSilence(payload); +} + +export const getSilenceMetaData = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.getSilenceMetadata(payload); +} + +export const updateSilence = async (silence: Record, payload: Record) => { + const { manager } = silence; + const service = getService(manager); + await service.updateSilence(silence, payload); +} + +export const deleteSilence = async (payload: Record) => { + const { id, manager } = payload; + const service = getService(manager); + await service.deleteSilence(id); +} + +export const deleteAlertByDataset = async (payload: Record) => { + try { + const { name } = payload; + const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) + for (const payload of alertRulePayload) { + await deleteAlertRule(payload, true) + await retireAlertSilence(_.get(payload, "id") || "") + } + return constants.ALERTS_RETIRED_SUCCESSFULLY; + } catch (error: any) { + throw new Error(constants.ALERTS_NOT_RETIRED); + } +} + +export const deleteMetricAliasByDataset = async (payload: Record) => { + try { + const { name } = payload; + const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) + if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) + for (const payload of metricAliasPayload) { + await payload.destroy() + } + return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; + } catch (error: any) { + throw new Error(constants.METRIC_ALIAS_NOT_DELETED); + } +} + +export const getAlertByDataset = async (payload: Record) => { + try { + const { name } = payload; + const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) + return alertRulePayload; + } catch (error) { + throw new Error(constants.ALERTS_FETCH_FAILURE); + } +} + +export const getAlertMetricsByDataset = async (payload: Record) => { + try { + const { name } = payload; + const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name }, raw: true }) + if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) + return metricAliasPayload; + } catch (error: any) { + throw new Error(constants.METRIC_ALIAS_NOT_DELETED); + } +} + +export const createAlertsByDataset = async (payload: any) => { + try { + for (const alerts of payload) { + const alertPayload = _.omit(alerts as any, ["id", "status", "createdAt", "updatedAt", "created_by", "updated_by"]) + await Alert.create(alertPayload) + } + } catch (error) { + throw new Error(constants.ALERT_CREATE_FAILURE); + } +} + +export const createMetricAliasByDataset = async (payload: any) => { + try { + for (const metrics of payload) { + const metricsPayload = _.omit(metrics as any, ["id", "createdAt", "updatedAt"]) + await Metrics.create(metricsPayload) + } + } catch (error) { + throw new Error(constants.METRIC_ALIAS_CREATE_FAILURE); + } +} + +export const publishAlertByDataset = async (payload: Record) => { + try { + const { name } = payload; + const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + if (!alertRulePayload) throw new Error("Alert rule does not exist") + for (const payload of alertRulePayload) { + await publishAlert(payload) + } + return constants.ALERTS_PUBLISHED_SUCCESSFULLY; + } catch (error: any) { + throw new Error(constants.ALERT_PUBLISH_FAILURE); + } +} + +export const retireAlertSilence = async (alert_id: string) => { + const silencePayload = await Silence.findOne({ where: { alert_id }, raw: true }); + if (silencePayload) { + await deleteSilence(silencePayload); + await Silence.destroy({ where: { alert_id } }); + } + return constants.SILENCE_DELETED_SUCCESSFULLY; +} \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/alert/index.ts b/api-service/src/services/managers/prometheus/alert/index.ts new file mode 100644 index 00000000..28f5412f --- /dev/null +++ b/api-service/src/services/managers/prometheus/alert/index.ts @@ -0,0 +1,17 @@ +import CONSTANTS from "../../constants" + +export const publishAlert = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const getAlerts = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const deleteAlert = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const generateAlertPayload = () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const deleteSystemRules= async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} diff --git a/api-service/src/services/managers/prometheus/index.ts b/api-service/src/services/managers/prometheus/index.ts new file mode 100644 index 00000000..f8f0da66 --- /dev/null +++ b/api-service/src/services/managers/prometheus/index.ts @@ -0,0 +1,5 @@ +import * as alertFunctions from "./alert" +import * as notificationFunctions from "./notification"; +import * as silenceFunctions from "./silences"; + +export default { ...alertFunctions, ...notificationFunctions, ...silenceFunctions } \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/notification/index.ts b/api-service/src/services/managers/prometheus/notification/index.ts new file mode 100644 index 00000000..9514d324 --- /dev/null +++ b/api-service/src/services/managers/prometheus/notification/index.ts @@ -0,0 +1,11 @@ +import CONSTANTS from "../../constants" + +export const createNotificationChannel = () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} +export const testNotificationChannel = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} +export const updateNotificationChannel = () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/silences/index.ts b/api-service/src/services/managers/prometheus/silences/index.ts new file mode 100644 index 00000000..60ed0aef --- /dev/null +++ b/api-service/src/services/managers/prometheus/silences/index.ts @@ -0,0 +1,24 @@ +import CONSTANTS from "../../constants"; + +const createSilence = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} + +const getSilenceMetadata = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} + +const updateSilence = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} + +const deleteSilence = async () => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} + +export { + createSilence, + getSilenceMetadata, + updateSilence, + deleteSilence +} \ No newline at end of file diff --git a/api-service/src/v2/services/telemetry.ts b/api-service/src/services/telemetry.ts similarity index 91% rename from api-service/src/v2/services/telemetry.ts rename to api-service/src/services/telemetry.ts index ede157f7..d4eca61b 100644 --- a/api-service/src/v2/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -2,17 +2,13 @@ import { Request, Response, NextFunction } from "express" import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; -import { Kafka } from "kafkajs"; +import {send} from "../connections/kafkaConnection" -const env = _.get(appConfig, "env") +const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); -const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const kafka = new Kafka({ clientId: telemetryTopic, brokers: brokerServers }); -const telemetryEventsProducer = kafka.producer(); -telemetryEventsProducer.connect().catch(err => console.error("Unable to connect to kafka", err.message)); const getDefaults = () => { return { @@ -29,7 +25,7 @@ const getDefaults = () => { sid: v4(), pdata: { id: `${env}.api.service`, - ver: "1.0.0" + ver: `${version}` } }, object: {}, @@ -54,7 +50,7 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { - telemetryEventsProducer.send({ topic: telemetryTopic, messages: [{ value: JSON.stringify(event) }] }).catch(console.log) + send({ messages: [{ value: JSON.stringify(event) }] }, telemetryTopic).catch(console.log); } const transformProps = (body: Record) => { diff --git a/api-service/src/v1/data/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts similarity index 84% rename from api-service/src/v1/data/telemetryActions.ts rename to api-service/src/telemetry/telemetryActions.ts index bbee6faa..befe6100 100644 --- a/api-service/src/v1/data/telemetryActions.ts +++ b/api-service/src/telemetry/telemetryActions.ts @@ -17,5 +17,8 @@ export default { "sqlQuery": "dataset:query:sql", "ingestEvents": "dataset:events:ingest", "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" + "datasetExhaust": "dataset:exhaust:get", + "copyDataset": "dataset:copy", + "readConnectors": "connectors:read", + "listConnectors": "connectors:list", } \ No newline at end of file diff --git a/api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts b/api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts new file mode 100644 index 00000000..832f4ff6 --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts @@ -0,0 +1,132 @@ +import app from "../../../app"; +import chai, { expect } from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { TestInputsForConnectorsList } from "./Fixtures"; +import { ConnectorRegistry } from "../../../models/ConnectorRegistry"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const apiId = "api.connectors.list" +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + + +describe("Connector List Api", () => { + afterEach(() => { + chai.spy.restore(); + }); + + it("Connectors list Success: With all the filters provided", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITH_BOTH_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list Success: Without filters", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITHOUT_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list Success: Filtered based on category", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_CATEGORY]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITH_CATEGORY_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_CATEGORY]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list Success: Filtered based on status", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_STATUS]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITH_STATUS_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_STATUS]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list failure: Invalid request payload provided", (done) => { + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.INVALID_REQUEST) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.code.should.be.eq("CONNECTORS_LIST_INPUT_INVALID") + expect(res.body.error.message).to.match(/^(.+) must NOT have fewer than 1 items$/) + done(); + }); + }); + + +}) \ No newline at end of file diff --git a/api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts b/api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts new file mode 100644 index 00000000..af996eca --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts @@ -0,0 +1,298 @@ +export const TestInputsForConnectorsList = { + INVALID_REQUEST: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { category: [] } + } + }, + REQUEST_WITHOUT_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": {} + }, + REQUEST_WITH_STATUS_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { status: ["Live"] } + } + }, + REQUEST_WITH_CATEGORY_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { + "category": [ + "Database" + ] + } + } + }, + REQUEST_WITH_BOTH_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { category: ["Database"], status: ["Live"] } + } + }, + + VALID_CONNECTORS_LIST:{ + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-07-30T15:17:51+05:30", + "params": { + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d", + "resmsgid": "fffa3ee0-da12-4bea-9b72-365571a62a4e" + }, + "responseCode": "OK", + "result": { + "data": [ + { + "id": "postgres-connector-1.0.0", + "connector_id": "postgres-connector", + "name": "PostgreSQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.732Z", + "updated_date": "2024-06-25T04:38:28.732Z" + }, + { + "id": "mysql-connector-1.0.0", + "connector_id": "mysql-connector", + "name": "MySQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.834Z", + "updated_date": "2024-06-25T04:38:28.834Z" + }, + { + "id": "oracle-connector-1.0.0", + "connector_id": "oracle-connector", + "name": "Oracle", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.840Z", + "updated_date": "2024-06-25T04:38:28.840Z" + }, + { + "id": "mssql-connector-1.0.0", + "connector_id": "mssql-connector", + "name": "MS SQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.847Z", + "updated_date": "2024-06-25T04:38:28.847Z" + }, + { + "id": "aws-s3-connector-0.1.0", + "connector_id": "aws-s3-connector", + "name": "AWS S3", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.237Z", + "updated_date": "2024-06-25T04:39:21.237Z" + }, + { + "id": "azure-blob-connector-0.1.0", + "connector_id": "azure-blob-connector", + "name": "Azure Blob Store", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.302Z", + "updated_date": "2024-06-25T04:39:21.302Z" + }, + { + "id": "gcs-connector-0.1.0", + "connector_id": "gcs-connector", + "name": "Google Cloud Storage", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The GCS Connector is used to move data from any Google Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.364Z", + "updated_date": "2024-06-25T04:39:21.364Z" + } + ], + "count": 7 + } + }, + VALID_CONNECTORS_LIST_CATEGORY:{ + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-07-30T15:07:42+05:30", + "params": { + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d", + "resmsgid": "632d3342-fd8a-47f7-afbb-96402a00b92f" + }, + "responseCode": "OK", + "result": { + "data": [ + { + "id": "aws-s3-connector-0.1.0", + "connector_id": "aws-s3-connector", + "name": "AWS S3", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.237Z", + "updated_date": "2024-06-25T04:39:21.237Z" + }, + { + "id": "azure-blob-connector-0.1.0", + "connector_id": "azure-blob-connector", + "name": "Azure Blob Store", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.302Z", + "updated_date": "2024-06-25T04:39:21.302Z" + }, + { + "id": "gcs-connector-0.1.0", + "connector_id": "gcs-connector", + "name": "Google Cloud Storage", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The GCS Connector is used to move data from any Google Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.364Z", + "updated_date": "2024-06-25T04:39:21.364Z" + } + ], + "count": 3 + } + }, + VALID_CONNECTORS_LIST_STATUS:{ + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-07-30T15:25:51+05:30", + "params": { + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d", + "resmsgid": "f506e725-eed4-41df-86dc-2477d5c4d19a" + }, + "responseCode": "OK", + "result": { + "data": [], + "count": 0 + } + } +} + diff --git a/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts b/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts new file mode 100644 index 00000000..e3247e3d --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts @@ -0,0 +1,117 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { ConnectorRegistry } from "../../../models/ConnectorRegistry"; +import { TestInputsForConnectorsRead } from "./Fixtures"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const apiId = "api.connectors.read" + +describe("Connectors Read API", () => { + afterEach(() => { + chai.spy.restore(); + }) + + it("Connector read success: When mode is not provided", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve(TestInputsForConnectorsRead.LIVE_CONNECTORS) + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-connector-1.0.0") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForConnectorsRead.LIVE_CONNECTORS)) + done(); + }); + }); + + it("Connector read success: When valid id is given, but value of mode is not provided", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve(TestInputsForConnectorsRead.LIVE_CONNECTORS) + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-connector-1.0.0?mode=") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForConnectorsRead.LIVE_CONNECTORS)) + done(); + }); + }); + + it("Connector read success: With mode=edit", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve(TestInputsForConnectorsRead.DRAFT_CONNECTORS) + }) + chai + .request(app) + .get("/v2/connectors/read/mssql-connector-2.0.0?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForConnectorsRead.DRAFT_CONNECTORS)) + done(); + }); + }); + + it("Connector read failure: when the requested connector of the id is not found", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-connector-1.0.0?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("Connector not found: postgres-connector-1.0.0") + res.body.error.code.should.be.eq("CONNECTOR_NOT_FOUND") + res.should.have.status(httpStatus.NOT_FOUND); + done(); + }); + }) + + + + it("Connector read Failure: when invalid id provided", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-conn") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("Connector not found: postgres-conn") + res.body.error.code.should.be.eq("CONNECTOR_NOT_FOUND") + res.should.have.status(httpStatus.NOT_FOUND); + done(); + }); + }); +}) diff --git a/api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts b/api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts new file mode 100644 index 00000000..cb0c504f --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts @@ -0,0 +1,284 @@ +export const TestInputsForConnectorsRead = { + LIVE_CONNECTORS: { + "id": "api.connectors.read", + "ver": "v2", + "ts": "2024-07-31T18:17:54+05:30", + "params": { + "status": "SUCCESS", + "resmsgid": "7587f564-c2d7-49a8-9e56-dc56f6808ced" + }, + "responseCode": "OK", + "result": { + "id": "postgres-connector-1.0.0", + "connector_id": "postgres-connector", + "name": "PostgreSQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg", + "status": "Live", + "ui_spec": { + "schema": { + "type": "object", + "properties": { + "connector_config": { + "title": "Connector Config", + "type": "object", + "encrypt": true, + "properties": { + "databaseType": { + "type": "string", + "title": "Database Type", + "enum": [ + "PostgreSQL", + "MySQL" + ], + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + }, + "dependencies": { + "databaseType": { + "oneOf": [ + { + "properties": { + "databaseType": { + "enum": [ + "PostgreSQL", + "MySQL" + ] + }, + "connection_info": { + "title": "Connection Information", + "type": "object", + "properties": { + "host": { + "type": "string", + "title": "Database Host", + "pattern": "/^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}(?:/[^\\s]*)?|localhost(?:/[^\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "port": { + "type": "number", + "title": "Database Port", + "minimum": 1, + "maximum": 65535, + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "name": { + "type": "string", + "title": "Database Name", + "pattern": "^[a-zA-Z0-9_]{1,64}$", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "username": { + "type": "string", + "title": "Database Username", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "password": { + "type": "string", + "title": "Database Password", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + } + }, + "schemaInfo": { + "title": "Schema Information", + "type": "object", + "properties": { + "table": { + "title": "Table Name", + "type": "string", + "pattern": "^[a-zA-Z_][a-zA-Z0-9_]{0,62}$", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "timestampColumn": { + "title": "Timestamp Column", + "type": "string", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + } + } + } + } + ] + } + } + }, + "operations_config": { + "title": "Operations Configuration", + "type": "object", + "properties": { + "batch_size": { + "type": "number", + "title": "Batch Size", + "default": 100, + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "max_batches": { + "type": "number", + "title": "Maximum Batches", + "default": 10, + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "pollingInterval": { + "type": "string", + "title": "Polling Interval", + "enum": [ + "Once", + "Periodic" + ], + "fieldDescription": [ + { + "type": "string", + "description": "Select polling interval" + } + ] + } + }, + "dependencies": { + "pollingInterval": { + "oneOf": [ + { + "properties": { + "pollingInterval": { + "enum": [ + "Periodic" + ] + }, + "schedule": { + "type": "string", + "title": "Schedule", + "enum": [ + "Hourly", + "Daily", + "Weekly", + "Monthly" + ], + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + }, + "required": [ + "schedule" + ] + } + ] + } + } + } + } + }, + "properties": { + "connector_config": { + "connection_info": { + "password": { + "ui:widget": "password" + } + } + }, + "operations_config": { + "batch_size": { + "ui:readonly": true + }, + "max_batches": { + "ui:readonly": true + } + } + } + }, + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.732Z", + "updated_date": "2024-06-25T04:38:28.732Z", + "live_date": "2024-06-25T04:38:28.732Z" + } + }, + + DRAFT_CONNECTORS: { + "id": "api.connectors.read", + "ver": "v2", + "ts": "2024-08-01T12:47:12+05:30", + "params": { + "status": "SUCCESS", + "resmsgid": "b6fcfb05-246c-4a1b-9eb1-27497ee9b80b" + }, + "responseCode": "OK", + "result": { + "id": "mssql-connector-2.0.0", + "connector_id": "mssql-connector", + "name": "MS SQL", + "type": "source", + "category": "Database", + "version": "2.0.0", + "description": "The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg", + "status": "Draft", + "ui_spec": {}, + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.847Z", + "updated_date": "2024-06-25T04:38:28.847Z", + "live_date": "2024-06-25T04:38:28.847Z" + } + } +} \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts similarity index 77% rename from api-service/src/v2/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts rename to api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts index a6037fff..1ff1c9b4 100644 --- a/api-service/src/v2/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import { TestInputsForDataIngestion } from "./Fixtures"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { Dataset } from "../../../models/Dataset"; import sinon from "sinon"; import { Kafka } from "kafkajs"; @@ -14,17 +14,16 @@ chai.should(); chai.use(chaiHttp); const kafka = new Kafka(connectionConfig.kafka.config); -const producer = kafka.producer(); const apiEndpoint = "/v2/data/in/:datasetId" const resultResponse = [ { - topicName: 'local.test.topic', + topicName: "local.test.topic", partition: 0, errorCode: 0, - baseOffset: '257', - logAppendTime: '-1', - logStartOffset: '0' + baseOffset: "257", + logAppendTime: "-1", + logStartOffset: "0" } ] const kafkaModule = require("../../../connections/kafkaConnection"); @@ -37,15 +36,13 @@ describe("DATA INGEST API", () => { it("it should ingest data for individual event", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: 'local.test.topic', - }, - extraction_config: { - is_batch_event: false, - extraction_key: "events", - batch_id: "id" - } + dataset_config: { + entry_topic: "local.test.topic", + }, + extraction_config: { + is_batch_event: false, + extraction_key: "events", + batch_id: "id" } }) }) @@ -72,10 +69,8 @@ describe("DATA INGEST API", () => { it("it should ingest data successfully", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: 'local.test.topic', - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -98,13 +93,37 @@ describe("DATA INGEST API", () => { }) }); + it("it should ingest data successfully v2", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ + api_version: "v2", + entry_topic: "local.test.topic", + }) + }) + const connectionStub = sinon.stub(kafkaModule, "connect").returns(true); + const sendStub = sinon.stub(kafkaModule, "send").returns(resultResponse); + chai + .request(app) + .post(apiEndpoint) + .send(TestInputsForDataIngestion.SAMPLE_INPUT_1) + .end((err, res) => { + res.should.have.status(200); + res.body.should.be.a("object"); + res.body.should.have.property("result"); + res.body.id.should.be.eq("api.data.in"); + res.body.result.message.should.be.eq("Data ingested successfully") + connectionStub.restore() + sendStub.restore() + chai.spy.restore(Dataset, "findOne") + done() + }) + }); + it("Failed to connect kafka.", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: 'local.test.topic', - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -126,9 +145,7 @@ describe("DATA INGEST API", () => { it("Entry topic not found", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: {} - } + dataset_config: {} }) }) @@ -196,8 +213,7 @@ describe("DATA INGEST API", () => { res.body.should.be.a("object") res.body.id.should.be.eq("api.data.in"); res.body.params.status.should.be.eq("FAILED"); - res.body.error.code.should.be.eq("DATA_INGESTION_FAILED"); - res.body.error.message.should.be.eq("Failed to ingest data") + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DataIngestTest/Fixtures.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DataIngestTest/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DataIngestTest/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts similarity index 95% rename from api-service/src/v2/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts rename to api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 3a7be3dc..9f5cf5b3 100644 --- a/api-service/src/v2/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import nock from "nock"; import { TestQueries } from "./Fixtures"; import { config } from "../../../configs/Config"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; +import chaiSpies from "chai-spies" +import { describe, it } from "mocha"; import { Datasource } from "../../../models/Datasource"; chai.use(chaiSpies) chai.should(); @@ -35,7 +35,7 @@ describe("QUERY API TESTS", () => { }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) - .reply(200, ['telemetry-events.1_rollup']) + .reply(200, ["telemetry-events.1_rollup"]) chai .request(app) .post("/v2/data/query/telemetry-events") @@ -89,8 +89,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); @@ -115,8 +115,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts new file mode 100644 index 00000000..c11de5f1 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts @@ -0,0 +1,23 @@ +export const TestQueries = { + VALID_QUERY: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"test\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2024-01-31/2024-02-01\"]},\"granularity\":\"week\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}]}}", + HIGH_LIMIT_NATIVE_QUERY: "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2024-11-31/2024-12-01\"]},\"granularity\":\"day\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}],\"limit\":10000,\"threshold\":10000}}", + WITHOUT_THRESOLD_QUERY: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\"},\"query\":{\"queryType\":\"timeBoundary\",\"dimension\":\"content_status\",\"metric\":\"count\",\"granularity\":\"all\",\"intervals\":[\"2020-12-21/2020-12-22\"],\"aggregations\":[]}}", + VALID_SQL_QUERY: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"test\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21' LIMIT 10\"}", + HIGH_LIMIT_SQL_QUERY: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\"},\"querySql\":{\"query\":\"SELECT msgid FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2021-01-22' LIMIT 100000\"}}", + HIGH_DATE_RANGE_SQL_QUERY: + `{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2022-02-12' LIMIT 10"}`, + LIMIT_IS_NAN: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"dataSource\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT content_status FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2021-01-12' LIMIT 100\"}", + DATASOURCE_NOT_FOUND: "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\"},\"query\":\"SELECT content_status FROM \\\"telemetry\\\" LIMIT 5\"}", + INVALID_DATE_RANGE_NATIVE: "{\"id\":\"api.data.out\",\"ver\":\"1.0\",\"ts\":\"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"table\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2023-01-31/2023-04-01\"]},\"granularity\":\"day\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"}]}}", + INVALID_SQL_QUERY: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"telemetry\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21' LIMIT 10\"}", + VALID_SQL_QUERY_WITHOUT_LIMIT: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21'\"}", + VALID_INTERVAL: + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":\"2024-01-31/2024-02-01\",\"granularity\":\"week\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}]}}", +} \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts similarity index 54% rename from api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 4bbeace3..51953ba0 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -1,15 +1,14 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; import { TestInputsForDatasetCreate, DATASET_CREATE_SUCCESS_FIXTURES, DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES } from "./Fixtures"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { sequelize } from "../../../connections/databaseConnection"; -import _ from "lodash"; import { apiId } from "../../../controllers/DatasetCreate/DatasetCreate" -import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { Dataset } from "../../../models/Dataset"; chai.use(spies); chai.should(); @@ -28,22 +27,13 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([{ nextVal: 9 }]) - }) - chai.spy.on(DatasourceDraft, "create", () => { - return Promise.resolve({}) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: { id: "telemetry" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - + chai .request(app) .post("/v2/datasets/create") @@ -66,9 +56,10 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) }) + chai .request(app) .post("/v2/datasets/create") @@ -79,7 +70,7 @@ describe("DATASET CREATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq(fixture.status) res.body.params.msgid.should.be.eq(fixture.msgid) - res.body.error.message.should.be.eq("Duplicate denorm key found") + res.body.error.message.should.be.eq("Duplicate denorm output fields found.") res.body.error.code.should.be.eq("DATASET_DUPLICATE_DENORM_KEY") done(); }); @@ -87,9 +78,6 @@ describe("DATASET CREATE API", () => { } it("Dataset creation failure: Invalid request payload provided", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) chai .request(app) .post("/v2/datasets/create") @@ -110,9 +98,7 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ datavalues: [] }) }) - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) + chai .request(app) .post("/v2/datasets/create") @@ -123,68 +109,11 @@ describe("DATASET CREATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset already exists") + res.body.error.message.should.be.eq("Dataset Already exists with id:sb-ddd") res.body.error.code.should.be.eq("DATASET_EXISTS") done(); }); }); - it("Dataset creation failure: When timestamp key does not exist in the data schema", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(null) - }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([{ nextVal: 9 }]) - }) - chai.spy.on(DatasetDraft, "create", () => { - return Promise.resolve({ dataValues: { id: "telemetry" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.DATASET_WITH_INVALID_TIMESTAMP) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Provided timestamp key not found in the data schema") - res.body.error.code.should.be.eq("DATASET_TIMESTAMP_NOT_FOUND") - done(); - }); - }); - - it("Dataset creation failure: Connection to the database failed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.DATASET_WITH_DUPLICATE_DENORM_KEY) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to create dataset") - res.body.error.code.should.be.eq("DATASET_CREATION_FAILURE") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts similarity index 73% rename from api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts index e5125b66..01b87ac1 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -1,5 +1,4 @@ import httpStatus from "http-status" -import _ from "lodash" export const TestInputsForDatasetCreate = { VALID_DATASET: { @@ -11,7 +10,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "validation_config": { "validate": true, @@ -44,29 +43,38 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "trip-details" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [] } }, - VALID_DATASET_WITH_DEFAULT_TS: { + VALID_DATASET_WITH_TRANSFORMATIONS: { "id": "api.datasets.create", - "ver": "v1", + "ver": "v2", "ts": "2024-04-10T16:10:50+05:30", "params": { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -75,6 +83,9 @@ export const TestInputsForDatasetCreate = { "eid": { "type": "string" }, + "ets": { + "type": "string" + }, "ver": { "type": "string" }, @@ -85,23 +96,33 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "obsrv_meta.syncts" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, + "transformations_config": [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], "tags": [] } }, - VALID_DATASET_WITH_TRANSFORMATIONS: { + VALID_DATASET_WITH_MULTIPLE_TRANSFORMATIONS: { "id": "api.datasets.create", - "ver": "v2", + "ver": "v1", "ts": "2024-04-10T16:10:50+05:30", "params": { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -110,10 +131,10 @@ export const TestInputsForDatasetCreate = { "eid": { "type": "string" }, - "ets": { + "ver": { "type": "string" }, - "ver": { + "ets": { "type": "string" }, "required": [ @@ -123,32 +144,24 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, - "transformations_config": [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ], + "transformations_config": [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, { "field_key": "ver", "transformation_function": { "type": "mask", "expr": "ver", "datatype": "string", "category": "pii" }, "mode": "Strict" }], "tags": [] } }, - VALID_DATASET_WITH_MULTIPLE_TRANSFORMATIONS: { + VALID_DATASET_WITH_CONNECTORS: { "id": "api.datasets.create", "ver": "v1", "ts": "2024-04-10T16:10:50+05:30", @@ -157,7 +170,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -179,42 +192,19 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] - }, - "transformations_config": [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ], + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "connectors_config":[{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, {"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5cai","connector_id":"debezium","connector_config":{"type":"debezium","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}], "tags": [] } }, @@ -228,7 +218,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -250,9 +240,18 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, @@ -265,7 +264,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -287,9 +286,18 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, VALID_MORE_THAN_MINIMAL_DATASET: { @@ -301,7 +309,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -325,14 +333,24 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master-telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, VALID_MORE_THAN_MINIMAL_MASTER_DATASET: { @@ -344,7 +362,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -368,14 +386,24 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - }, + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } } }, VALID_MASTER_DATASET: { @@ -386,9 +414,8 @@ export const TestInputsForDatasetCreate = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "id": "sb-telemetry2", "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "validation_config": { "validate": true, @@ -421,13 +448,23 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [] } @@ -445,41 +482,6 @@ export const TestInputsForDatasetCreate = { } }, - DATASET_WITH_INVALID_TIMESTAMP: { - "id": "api.datasets.create", - "ver": "v1", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "dataset_id": "sb-ddd", - "type": "dataset", - "name": "sb-telemetry2", - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "dataset_config": { - "data_key": "", - "timestamp_key": "lastAccessed" - }, - "tags": [] - } - }, - DATASET_WITH_DUPLICATE_DENORM_KEY: { "id": "api.datasets.create", "ver": "v2", @@ -488,9 +490,8 @@ export const TestInputsForDatasetCreate = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "id": "sb-telemetry2", "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -512,11 +513,13 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" }, { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] } @@ -582,19 +585,19 @@ export const DATASET_CREATE_SUCCESS_FIXTURES = [ "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, { - "title": "Dataset creation success: When multiple transformation payload provided with same field key", - "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_MULTIPLE_TRANSFORMATIONS, + "title": "Dataset creation success: When connectors payload provided", + "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_CONNECTORS, "httpStatus": httpStatus.OK, "status": "SUCCESS", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, { - "title": "Dataset creation success: Geenerating ingestion spec successfully using the data schema", - "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_DEFAULT_TS, + "title": "Dataset creation success: When multiple transformation payload provided with same field key", + "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_MULTIPLE_TRANSFORMATIONS, "httpStatus": httpStatus.OK, "status": "SUCCESS", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, + } ] export const DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES = [ @@ -604,12 +607,5 @@ export const DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES = [ "httpStatus": httpStatus.BAD_REQUEST, "status": "FAILED", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - { - "title": "Master Dataset creation failure: Dataset contains duplicate denorm out field", - "requestPayload": _.set(TestInputsForDatasetCreate.DATASET_WITH_DUPLICATE_DENORM_KEY, "request.type", "master-dataset"), - "httpStatus": httpStatus.BAD_REQUEST, - "status": "FAILED", - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" } ] \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts new file mode 100644 index 00000000..380ee8ad --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts @@ -0,0 +1,119 @@ +import app from "../../../app"; +import chai, { expect } from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { apiId } from "../../../controllers/DatasetList/DatasetList"; +import { TestInputsForDatasetList } from "./Fixtures"; +import { Dataset } from "../../../models/Dataset"; +import { DatasetDraft } from "../../../models/DatasetDraft"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + +describe("DATASET LIST API", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset list success: When no filters are provided", (done) => { + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) + }) + chai.spy.on(DatasetDraft, "findAll", () => { + return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) + }) + chai + .request(app) + .post("/v2/datasets/list") + .send(TestInputsForDatasetList.REQUEST_WITHOUT_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(2) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify(TestInputsForDatasetList.VALID_RESPONSE) + result.should.be.eq(expectedResult) + done(); + }); + }); + + it("Dataset list success: When status filter provided in request payload", (done) => { + chai.spy.on(DatasetDraft, "findAll", () => { + return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/list") + .send(TestInputsForDatasetList.REQUEST_WITH_STATUS_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA }]) + result.should.be.eq(expectedResult) + done(); + }); + }); + + it("Dataset list success: When type filter provided in request payload", (done) => { + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) + }) + chai.spy.on(DatasetDraft, "findAll", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/list") + .send(TestInputsForDatasetList.REQUEST_WITH_TYPE_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA}]) + result.should.be.eq(expectedResult) + done(); + }); + }); + + it("Dataset list failure: Invalid request payload provided", (done) => { + chai + .request(app) + .post("/v2/datasets/list") + .send(TestInputsForDatasetList.INVALID_REQUEST) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.code.should.be.eq("DATASET_LIST_INPUT_INVALID") + expect(res.body.error.message).to.match(/^(.+) must be equal to one of the allowed values$/) + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetList/Fixtures.ts new file mode 100644 index 00000000..31a5ba1c --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetList/Fixtures.ts @@ -0,0 +1,96 @@ +export const TestInputsForDatasetList = { + VALID_DRAFT_DATASET_SCHEMA: { + "dataset_id": "telemetry", + "name": "telemetry", + "type": "events", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "tags": [ + "tag1", + "tag2" + ], + "status": "Draft", + "version": 1, + "api_version": "v2" + }, + VALID_LIVE_DATASET_SCHEMA: { + "dataset_id": "sb-telemetry", + "name": "sb-telemetry", + "type": "master", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "tags": [ + "tag1", + "tag2" + ], + "status": "Live", + "data_version": 1, + "api_version": "v2" + }, + + REQUEST_WITHOUT_FILTERS: { + "id": "api.datasets.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": {} + }, + REQUEST_WITH_STATUS_FILTERS: { + "id": "api.datasets.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { status: ["Draft"] } + } + }, + REQUEST_WITH_TYPE_FILTERS: { + "id": "api.datasets.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { status: "Live", type: "master" } + } + }, + INVALID_REQUEST: { + "id": "api.datasets.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { status: ["Ready"] } + } + }, + VALID_RESPONSE: [{"dataset_id":"sb-telemetry","name":"sb-telemetry","type":"master","dataset_config":{"indexing_config":{"olap_store_enabled":false,"lakehouse_enabled":true,"cache_enabled":true},"keys_config":{"data_key":"ets"},"file_upload_path":["telemetry.json"]},"tags":["tag1","tag2"],"status":"Live","data_version":1,"api_version":"v2"},{"dataset_id":"telemetry","name":"telemetry","type":"events","dataset_config":{"indexing_config":{"olap_store_enabled":false,"lakehouse_enabled":true,"cache_enabled":false},"keys_config":{"timestamp_key":"ets"},"file_upload_path":["telemetry.json"]},"tags":["tag1","tag2"],"status":"Draft","version":1,"api_version":"v2"}] +} \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts new file mode 100644 index 00000000..c3e766e7 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -0,0 +1,355 @@ +import app from "../../../app"; +import chai, { expect } from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { apiId, defaultFields } from "../../../controllers/DatasetRead/DatasetRead"; +import { TestInputsForDatasetRead } from "./Fixtures"; +import { DatasetTransformations } from "../../../models/Transformation"; +import { Dataset } from "../../../models/Dataset"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; +import { ConnectorInstances } from "../../../models/ConnectorInstances"; +import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; +import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; +import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +describe("DATASET READ API", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset read success: When minimal fields requested", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "name": "sb-telemetry", "version": 1 }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?fields=name,version,connectors_config,transformations_config") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.name.should.be.eq("sb-telemetry") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify({ "name": "sb-telemetry", "version": 1, "connectors_config": [], "transformations_config": [] })) + done(); + }); + }); + + it("Dataset read success: Fetch all dataset fields when fields param is empty", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.DRAFT_SCHEMA) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.type.should.be.eq("event") + res.body.result.status.should.be.eq("Draft") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA })) + done(); + }); + }); + + it("Dataset read success: Fetch live dataset when mode param not provided", (done) => { + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.status.should.be.eq("Live") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA, connectors_config: TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2, transformations_config: TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA })) + done(); + }); + }); + + it("Dataset read success: Creating draft on mode=edit if no draft found in v2", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.MASTER_DATASET_SCHEMA) + }) + chai.spy.on(DatasetDraft, "create", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.name.should.be.eq("sb-telemetry") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) + done(); + }); + }); + + it("Dataset read success: Creating draft on mode=edit if no draft found in v1", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.MASTER_DATASET_SCHEMA) + }) + chai.spy.on(DatasetDraft, "create", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.name.should.be.eq("sb-telemetry") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) + done(); + }); + }); + + it("Dataset read success: Migrating v1 draft dataset to v2 on mode=edit", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.DRAFT_SCHEMA_V1) + }) + chai.spy.on(DatasetTransformationsDraft, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfigDraft, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) + }) + chai.spy.on(DatasetTransformationsDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfigDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasourceDraft, "destroy", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.name.should.be.eq("sb-telemetry") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(_.pick(TestInputsForDatasetRead.DRAFT_SCHEMA_V1, defaultFields))) + done(); + }); + }); + + it("Dataset read failure: Updating dataset status to draft on mode=edit fails as live record not found", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("Dataset with the given dataset_id:sb-telemetry not found") + res.body.error.code.should.be.eq("DATASET_NOT_FOUND") + done(); + }); + }); + + it("Dataset read failure: When dependent denorm master dataset not found", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("The dependent dataset not found") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_FOUND") + done(); + }); + }); + + it("Dataset read failure: When dependent denorm master dataset not live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "dataset_id": "master_dataset", "dataset_config": { "cache_config": { "redis_db": 20 } } }]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.PRECONDITION_REQUIRED); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("The dependent master dataset is not published") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_LIVE") + done(); + }); + }); + + it("Dataset read failure: When the dataset of requested dataset_id not found", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?fields=name") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("Dataset with the given dataset_id:sb-telemetry not found") + res.body.error.code.should.be.eq("DATASET_NOT_FOUND") + done(); + }); + }); + + it("Dataset read failure: When specified field of live dataset cannot be found", (done) => { + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?fields=data") + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + expect(res.body.error.message).to.match(/^The specified field(.+) in the dataset cannot be found.$/) + res.body.error.code.should.be.eq("DATASET_INVALID_FIELDS") + done(); + }); + }); + + it("Dataset read failure: When specified field of draft dataset cannot be found", (done) => { + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?fields=data&mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + expect(res.body.error.message).to.match(/^The specified field(.+) in the dataset cannot be found.$/) + res.body.error.code.should.be.eq("DATASET_INVALID_FIELDS") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts new file mode 100644 index 00000000..d520a102 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -0,0 +1,143 @@ +export const TestInputsForDatasetRead = { + DRAFT_SCHEMA: { + "dataset_id": "sb-telemetry", + "name": "sb-telemetry", + "type": "event", + "status": "Draft", + "tags": [ + "tag1", + "tag2" + ], + "version": 1, + "api_version": "v2", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } + }, + DRAFT_SCHEMA_V1: { + "dataset_id": "sb-telemetry", + "name": "sb-telemetry", + "type": "event", + "status": "Draft", + "tags": [ + "tag1", + "tag2" + ], + "validation_config": { + "validate": true, + "validation_mode": "Strict", + "mode": "Strict" + }, + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "eid": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ets": { + "type": "string" + }, + "required": [ + "eid" + ] + }, + "additionalProperties": true + }, + "version": 1, + "api_version": "v1", + "dataset_config": { + "timestamp_key": "ets", + "data_key": "", + "redis_db_host": "localhost", + "redis_db_port": 6379, + "redis_db": 0 + } + }, + LIVE_SCHEMA: { + "dataset_id": "sb-telemetry", + "name": "sb-telemetry", + "type": "event", + "status": "Live", + "tags": [ + "tag1", + "tag2" + ], + "data_version": 1, + "api_version": "v2", + "denorm_config": { + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "redis_db": 16 + } + ] + }, + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } + }, + MASTER_DATASET_SCHEMA:[{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":16}}}], + TRANSFORMATIONS_SCHEMA: [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + TRANSFORMATIONS_SCHEMA_V1: [ + { + "field_key": "eid", + "transformation_function": { + "type": "mask", + "expr": "eid", + "condition": null + }, + "mode": "Strict", + "metadata": { + "_transformationType": "mask", + "_transformedFieldDataType": "string", + "_transformedFieldSchemaType": "string", + "section": "transformation" + } + } + ], + CONNECTORS_SCHEMA_V1: [ + { + "id": "hsh882ehdshe", + "connector_type": "kafka", + "connector_config": { + "topic": "local.ingest", + "brokerURL": "localhost:9092" + } + } + ], + CONNECTORS_SCHEMA_V2: [ + { + "id": "hsh882ehdshe", + "connector_id": "kafka", + "connector_config": { + "topic": "local.ingest", + "brokerURL": "localhost:9092" + } + } + ] +} \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts new file mode 100644 index 00000000..0016ae5d --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts @@ -0,0 +1,86 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { TestInputsForDatasetStatusTransition } from "./Fixtures"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; +import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { sequelize } from "../../../connections/databaseConnection"; + + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" + +describe("DATASET STATUS TRANSITION DELETE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset status transition success: When the action is to Delete draft datasets", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ dataset_id: "telemetry", status: "Draft", id: "telemetry.1" }) + }) + chai.spy.on(DatasetTransformationsDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfigDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasourceDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetDraft, "destroy", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Delete successful") + res.body.result.dataset_id.should.be.eq("telemetry.1") + done(); + }); + }); + + it("Dataset status transition failure: When dataset is not found to delete", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") + res.body.error.code.should.be.eq("DATASET_NOT_FOUND") + done(); + }); + }); +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts new file mode 100644 index 00000000..e26ddafe --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -0,0 +1,336 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { TestInputsForDatasetStatusTransition } from "./Fixtures"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import { commandHttpService } from "../../../connections/commandServiceConnection"; +import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { Dataset } from "../../../models/Dataset"; +import { Datasource } from "../../../models/Datasource"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" + +describe("DATASET STATUS TRANSITION LIVE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset status transition success: When the action is to set dataset live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is to set dataset live v1 by creating hudi spec", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is to set dataset live v2 by updating hudi spec", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "api_version":"v2", "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "findOne", () => { + return Promise.resolve({"ingestion_spec":{"dataset": "dataset-all-fields4", "schema": {"table": "dataset-all-fields4_events", "partitionColumn": "eid", "timestampColumn": "obsrv_meta.syncts", "primaryKey": "eid", "columnSpec": [{"type": "string", "name": "mid", "index": 1}, {"type": "epoch", "name": "ets", "index": 2}, {"type": "string", "name": "userdata.mid", "index": 3}, {"type": "epoch", "name": "userdata.ets", "index": 4}, {"type": "string", "name": "userdata.eid", "index": 5}, {"type": "string", "name": "email", "index": 6}, {"type": "string", "name": "obsrv.meta.source.connector", "index": 7}, {"type": "string", "name": "obsrv.meta.source.id", "index": 8}]}, "inputFormat": {"type": "json", "flattenSpec": {"fields": [{"type": "path", "expr": "$.mid", "name": "mid"}, {"type": "path", "expr": "$.ets", "name": "ets"}, {"type": "path", "expr": "$.eid", "name": "eid"}, {"type": "path", "expr": "$.userdata.mid", "name": "userdata.mid"}, {"type": "path", "expr": "$.userdata.ets", "name": "userdata.ets"}, {"type": "path", "expr": "$.userdata.eid", "name": "userdata.eid"}, {"type": "path", "expr": "$.email", "name": "email"}, {"type": "path", "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts"}, {"type": "path", "expr": "$.obsrv_meta.source.connector", "name": "obsrv.meta.source.connector"}, {"type": "path", "expr": "$.obsrv_meta.source.connectorInstance", "name": "obsrv.meta.source.id"}]}}}}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") + res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") + done(); + }); + }); + + it("Dataset status transition success: When the action is to set master dataset live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([[{ nextval: 9 }]]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("master-telemetry") + done(); + }); + }); + + it("Dataset status transition failure: When the dependent denorm master dataset is not live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(_.clone(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH)) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Retired", "dataset_config": { "redis_db": 21 }, "api_version": "v1" }]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.PRECONDITION_REQUIRED); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("The datasets with id:master-dataset are not in published status") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_LIVE") + done(); + }); + }); + + it("Dataset status transition failure: When dataset to publish is self referencing the denorm master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({...TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH, "id": "master-dataset"}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.CONFLICT); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("The denorm master dataset is self-referencing itself") + res.body.error.code.should.be.eq("SELF_REFERENCING_MASTER_DATA") + done(); + }); + }); + + it("Dataset status transition failure: When dataset is not found to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") + res.body.error.code.should.be.eq("DATASET_NOT_FOUND") + done(); + }) + }) + + it("Dataset status transition failure: When the command api call to publish dataset fails", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.reject() + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + done(); + }); + }); + + it("Dataset status transition failure: When the dataset to publish is in draft state", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({...TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH, status: "Draft"}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.CONFLICT); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.error.code.should.be.eq("DATASET_LIVE_FAILURE") + res.body.error.message.should.be.eq("Transition failed for dataset: telemetry status:Draft with status transition to Live") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts new file mode 100644 index 00000000..f7d5f1f3 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -0,0 +1,133 @@ +import app from "../../../app"; +import chai, { expect } from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { TestInputsForDatasetStatusTransition } from "./Fixtures"; +import { DatasetDraft } from "../../../models/DatasetDraft"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" + +describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset status transition success: When the action is make dataset ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_READY_TO_PUBLISH) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is make master dataset ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_MASTER_SCHEMA_FOR_READY_TO_PUBLISH) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition failure: When dataset is not found to ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") + res.body.error.code.should.be.eq("DATASET_NOT_FOUND") + done(); + }); + }); + + it("Dataset status transition failure: When dataset is already ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_READY_TO_PUBLISH, "status": "ReadyToPublish" }) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.CONFLICT); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Transition failed for dataset: dataset-all-fields7 status:ReadyToPublish with status transition to ReadyToPublish") + res.body.error.code.should.be.eq("DATASET_READYTOPUBLISH_FAILURE") + done(); + }); + }); + + + it("Dataset status transition failure: Configs invalid to set status to ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.INVALID_SCHEMA_FOR_READY_TO_PUBLISH) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + expect(res.body.error.message).to.match(/^#required must have(.+)/) + res.body.error.code.should.be.eq("DATASET_CONFIGS_INVALID") + done(); + }); + }); +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts new file mode 100644 index 00000000..1a0428af --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -0,0 +1,234 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { TestInputsForDatasetStatusTransition } from "./Fixtures"; +import { Dataset } from "../../../models/Dataset"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import { DatasetTransformations } from "../../../models/Transformation"; +import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; +import { Datasource } from "../../../models/Datasource"; +import { commandHttpService } from "../../../connections/commandServiceConnection"; +import { druidHttpService } from "../../../connections/druidConnection"; +import { sequelize } from "../../../connections/databaseConnection"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" + +describe("DATASET STATUS TRANSITION RETIRE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset status transition success: When the action is to Retire dataset", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE) + }) + chai.spy.on(DatasetTransformations, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfig, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve([{ datasource_ref: "telemetry" }]) + }) + chai.spy.on(druidHttpService, "post", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Retire successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is to Retire master dataset", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "master-dataset" }) + }) + chai.spy.on(DatasetTransformations, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfig, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Retire successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition successs: Dataset successfully retired on delete supervisors failure", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) + }) + chai.spy.on(DatasetTransformations, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfig, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve(["telemetry"]) + }) + chai.spy.on(druidHttpService, "post", () => { + return Promise.reject({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Retire successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition failure: When dataset is not found to retire", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") + res.body.error.code.should.be.eq("DATASET_NOT_FOUND") + done(); + }) + }) + + it("Dataset status transition failure: When dataset is already retired", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE, status: "Retired" }) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) + .end((err, res) => { + res.should.have.status(httpStatus.CONFLICT); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Transition failed for dataset: dataset-all-fields7 status:Retired with status transition to Retire") + res.body.error.code.should.be.eq("DATASET_RETIRE_FAILURE") + done(); + }) + }) + + it("Dataset status transition failure: When dataset to retire is used by other datasets", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE, type: "master" }) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "dataset-all-fields7" }] } }]) + }) + chai.spy.on(DatasetDraft, "findAll", () => { + return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "dataset-all-fields7" }] } }]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset") + res.body.error.code.should.be.eq("DATASET_IN_USE") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts new file mode 100644 index 00000000..7117aa94 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts @@ -0,0 +1,60 @@ +import app from "../../../app"; +import chai, { expect } from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import _ from "lodash"; +import { TestInputsForDatasetStatusTransition } from "./Fixtures"; +import { DatasetDraft } from "../../../models/DatasetDraft"; + + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" + +describe("DATASET STATUS TRANSITION API", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Dataset status transition failure: Invalid request payload provided", (done) => { + + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.INVALID_SCHEMA) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + expect(res.body.error.message).to.match(/^#properties\/request(.+)$/) + res.body.error.code.should.be.eq("DATASET_STATUS_TRANSITION_INVALID_INPUT") + done(); + }); + }); + + it("Dataset status transition failure: When the action is performed on v1 apis", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ api_version: "v1" }) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.error.code.should.be.eq("DATASET_API_VERSION_MISMATCH") + res.body.error.message.should.be.eq("Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset") + done(); + }); + }); +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts new file mode 100644 index 00000000..e65f1852 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -0,0 +1,164 @@ +export const TestInputsForDatasetStatusTransition = { + VALID_SCHEMA_FOR_DELETE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry.1", + "status": "Delete" + } + }, + VALID_SCHEMA_FOR_LIVE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry", + "status": "Live" + } + }, + VALID_SCHEMA_FOR_LIVE_MASTER: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "master-telemetry", + "status": "Live" + } + }, + VALID_SCHEMA_FOR_RETIRE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry", + "status": "Retire" + } + }, + INVALID_SCHEMA: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry.1", + "status": "" + } + }, + VALID_REQUEST_FOR_READY_FOR_PUBLISH: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry", + "status": "ReadyToPublish" + } + }, + VALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "data_key": "eid", "partition_key": "eid", "timestamp_key": "obsrv_meta.syncts" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": [] }, + "tags": ["tag1"], + "status": "Draft", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-07-24 19:12:13.021", + "updated_date": "2024-07-25 06:12:38.412", + "version_key": "1721887933020", + "api_version": "v2", + "transformations_config": [{ "field_key": "email", "transformation_function": { "type": "mask", "expr": "mid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + "connectors_config": [{ "id": "91898e828u82882u8", "connector_id": "kafka", "connector_config": "AR/hz8iBxRyc9s0ohXa3+id+7GoWtjVjNWvurFFgV1Ocw2kgc+XVbnfXX26zkP3+rQ49gio0JzwsFzOK61TtXLx968IKol5eGfaEHF68O5faoxxjKBsyvhPaRQ91DKKi", "version": "v1" }], + "sample_data": {}, + "entry_topic": "local.ingest" + }, + VALID_MASTER_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "master", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "data_key": "eid", "partition_key": "eid", "timestamp_key": "obsrv_meta.syncts" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": [] }, + "tags": ["tag1"], + "status": "Draft", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-07-24 19:12:13.021", + "updated_date": "2024-07-25 06:12:38.412", + "version_key": "1721887933020", + "transformations_config": [], + "connectors_config": [], + "api_version": "v2", + "sample_data": {}, + "entry_topic": "local.ingest" + }, + INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "tags": ["tag1"], + "status":"Draft", + "version_key": "1721887933020", + "api_version": "v2" + }, + SCHEMA_TO_RETIRE: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "tags": ["tag1"], + "status":"Live", + "version_key": "1721887933020", + "api_version": "v2" + }, + DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "ets", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "master-telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_MASTER_DATASET_INVALID: { "dataset_id": "master-telemetry", "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } +} \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts new file mode 100644 index 00000000..1122aac4 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts @@ -0,0 +1,74 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import _ from "lodash"; +import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; +import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +describe("DATASET CONNECTORS UPDATE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Success: Dataset connectors successfully added", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", connectors_config:[] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_CONNECTORS_ADD) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + + it("Success: Dataset connectors successfully removed", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", connectors_config:[{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_CONNECTORS_REMOVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts similarity index 83% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index 7c54adf2..eaa80648 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -1,14 +1,13 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; import { apiId, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -23,18 +22,12 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Success: Dataset dedupe configs updated with dedup key if duplicates need to be dropped", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -55,22 +48,16 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Success: Dataset dedupe configs updated with default values if duplicates need to be dropped", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, dedup_config: { drop_duplicates: false } } }) + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, dedup_config: { drop_duplicates: false, dedup_key: "mid" } } }) .end((err, res) => { console.log(res.body.result) res.should.have.status(httpStatus.OK); @@ -88,9 +75,6 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Failure: Dedup key not provided when duplicates need to be dropped", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts similarity index 52% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index d0aa3f84..5f330032 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -1,44 +1,37 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); chai.use(chaiHttp); -describe("DATASET TAGS UPDATE", () => { +describe("DATASET DENORM UPDATE", () => { afterEach(() => { chai.spy.restore(); }); - it("Success: Dataset tags successfully added", (done) => { + it("Success: Dataset denorms successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, denorm_config: { denorm_fields: [] } + id: "telemetry", status: "Draft", type:"event", version_key: validVersionKey, api_version:"v2", denorm_config: { denorm_field: [] } }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TAG_ADD) + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DENORM_ADD) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -52,25 +45,19 @@ describe("DATASET TAGS UPDATE", () => { }); }); - it("Success: Dataset tags successfully removed", (done) => { + it("Success: Dataset denorms successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"] + id: "telemetry", status: "Draft", type:"event", version_key: validVersionKey, api_version:"v2", denorm_config: { denorm_fields: [{ denorm_out_field: "userdata" }] } }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TAG_REMOVE) + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DENORM_REMOVE) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -84,25 +71,25 @@ describe("DATASET TAGS UPDATE", () => { }); }); - it("Success: When payload contains same tags to be added or removed", (done) => { + it("Success: When payload contains same denorms to be removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"] + id: "telemetry", version_key: validVersionKey, type:"dataset", api_version:"v2", status: "Draft", denorm_config: { + denorm_fields: [{ + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }] + } }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_TAGS_ADD) + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_DENORM_REMOVE) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -116,58 +103,34 @@ describe("DATASET TAGS UPDATE", () => { }); }); - it("Failure: When tags provided to add already exists", (done) => { + it("Success: Ignore the denorm when payload contains denorm that already exists", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag3", "tag1"] + id: "telemetry", version_key: validVersionKey, type:"dataset", api_version:"v2", status: "Draft", denorm_config: { + denorm_fields: [ { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }] + } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset tags already exist") - res.body.error.code.should.be.eq("DATASET_TAGS_EXISTS") - done(); - }); - }); - - it("Failure: When tags provided to delete does not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag5"] - }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) chai .request(app) .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_EXISTING_DENORM) .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); + res.should.have.status(httpStatus.OK); res.body.should.be.a("object") res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") + res.body.params.status.should.be.eq("SUCCESS") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset tags do not exist to remove") - res.body.error.code.should.be.eq("DATASET_TAGS_DO_NOT_EXIST") + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") done(); }); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts similarity index 82% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 2a4a12ce..d17f1c3b 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; @@ -23,18 +23,13 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Success: Dataset extraction configs updated if it is a batch event", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, api_version: "v2", type: "event" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -55,22 +50,29 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Success: Dataset extraction configs updated with default values if it is not batch event", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, api_version: "v2", type: "event" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, extraction_config: { "is_batch_event": false } } }) + .send({ + ...requestStructure, request: { + dataset_id: "telemetry", version_key: validVersionKey, + "extraction_config": { + "is_batch_event": false, + "extraction_key": "events", + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "id" + } + } + } + }) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -86,9 +88,6 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Failure: Extraction configs not provided as batch event is true", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts new file mode 100644 index 00000000..49ba7b8a --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -0,0 +1,77 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import _ from "lodash"; +import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; +import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { sequelize } from "../../../connections/databaseConnection"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +describe("DATASET TAGS UPDATE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Success: Dataset tags successfully added", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, denorm_config: { denorm_fields: [] }, api_version: "v2" + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TAG_ADD) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + + it("Success: Dataset tags successfully removed", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], api_version: "v2" + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TAG_REMOVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts new file mode 100644 index 00000000..58252e83 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -0,0 +1,102 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import _ from "lodash"; +import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; +import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { sequelize } from "../../../connections/databaseConnection"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +describe("DATASET TRANSFORMATIONS UPDATE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Success: Dataset transformations successfully added", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", "transformations_config":[] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_ADD) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + + it("Success: Dataset transformations successfully removed", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", "transformations_config":[{ "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_REMOVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + + it("Success: When payload contains same transformation field_key to be added, updated or removed", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_TRANSFORMATION_ADD_REMOVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts similarity index 67% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 6239d93f..c2050f23 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -1,16 +1,16 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { apiId, errorCode, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { apiId, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" import { sequelize } from "../../../connections/databaseConnection"; -import { DatasourceDraft } from "../../../models/DatasourceDraft"; + chai.use(spies); chai.should(); @@ -24,17 +24,12 @@ describe("DATASET UPDATE API", () => { it("Dataset updation success: When minimal request payload provided", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -53,40 +48,20 @@ describe("DATASET UPDATE API", () => { }); it("Dataset updation success: When full request payload provided", (done) => { - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid" }] - } + }, api_version: "v2" }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) + chai .request(app) .patch("/v2/datasets/update") @@ -105,9 +80,7 @@ describe("DATASET UPDATE API", () => { }); it("Dataset updation failure: When no fields with dataset_id is provided in the request payload", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -128,9 +101,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -141,7 +112,7 @@ describe("DATASET UPDATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset does not exists to update") + res.body.error.message.should.be.eq("Dataset does not exists with id:telemetry") res.body.error.code.should.be.eq("DATASET_NOT_EXISTS") done(); }); @@ -149,11 +120,9 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: When dataset to update is outdated", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: "1813444815918" }) - }) - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) + return Promise.resolve({ id: "telemetry", type: "event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) }) + chai .request(app) .patch("/v2/datasets/update") @@ -170,13 +139,32 @@ describe("DATASET UPDATE API", () => { }); }); - it("Dataset updation failure: Dataset to update is not in draft state", (done) => { + it("Dataset updation failure: When dataset to update is of api_version v1", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ status: "Live" }) + return Promise.resolve({ id: "telemetry", type: "event", status: "Draft", version_key: "1813444815918", api_version: "v1" }) }) - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) + + chai + .request(app) + .patch("/v2/datasets/update") + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, name: "telemetry" } }) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset") + res.body.error.code.should.be.eq("DATASET_API_VERSION_MISMATCH") + done(); + }); + }); + + it("Dataset updation failure: Dataset to update is not in draft state", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ id: "telemetry", type: "event", status: "Live", version_key: "1713444815918", api_version: "v2" }) }) + chai .request(app) .patch("/v2/datasets/update") @@ -197,12 +185,6 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.reject({}) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -213,8 +195,7 @@ describe("DATASET UPDATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to update dataset") - res.body.error.code.should.be.eq(errorCode) + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR") done(); }); }); @@ -227,17 +208,11 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset name updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -256,9 +231,7 @@ describe("DATASET UPDATE API", () => { }); it("Failure: Failed to update the dataset name", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -285,56 +258,12 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset data schema updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey - }) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DATA_SCHEMA_VALID) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: Ingestion spec updateded successfully", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -352,38 +281,7 @@ describe("DATASET UPDATE API", () => { }); }); - it("Failure: When timestamp key does not exist in the data schema", (done) => { - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_WITH_INVALID_TIMESTAMP) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Provided timestamp key not found in the data schema") - res.body.error.code.should.be.eq("DATASET_TIMESTAMP_NOT_FOUND") - done(); - }); - }); - it("Failure: Failed to update data schema", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -410,18 +308,12 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset config updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -440,9 +332,6 @@ describe("DATASET UPDATE API", () => { }); it("Failure: Failed to update dataset config", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts similarity index 84% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index fedfaa1d..d5feed17 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; @@ -23,18 +23,13 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Success: Dataset validation configs updated when validation is true", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -55,22 +50,17 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Success: Dataset validation configs updated with default values when validation is false", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, validation_config: { "validate": false } } }) + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, validation_config: { "validate": false, "mode": "Strict" } } }) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -86,9 +76,6 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Failure: Validation configs not provided as validation is true", (done) => { - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts similarity index 63% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 4dd1455e..ae9628a4 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -15,7 +15,8 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "name": "telemetry" + "name": "telemetry", + "sample_data":{"events":{}} } }, @@ -25,11 +26,8 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": [ - "tag1", - "tag2" - ], - "action": "add" + "value": "tag1", + "action": "upsert" }] } }, @@ -40,10 +38,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": [ - "tag1", - "tag2" - ], + "value": "tag1", "action": "remove" }] } @@ -56,11 +51,12 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master" }, - "action": "add" + "action": "upsert" } ] } @@ -74,7 +70,7 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, @@ -89,16 +85,23 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "add" - }] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }], + } + }, + + DATASET_UPDATE_CONNECTORS_ADD: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "connectors_config":[{"value":{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, "action": "upsert"}], + } + }, + + DATASET_UPDATE_CONNECTORS_REMOVE: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "connectors_config":[{"value":{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, "action": "upsert"}], } }, @@ -198,8 +201,18 @@ export const TestInputsForDatasetUpdate = { "dataset_id": "telemetry", "version_key": validVersionKey, "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets", + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] } } }, @@ -208,16 +221,7 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }], } }, @@ -282,69 +286,46 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master" }, - "action": "add" + "action": "upsert" }, { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" } ] }, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "add" + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "key2", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "remove" }], + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" + "keys_config": { + "timestamp_key": "ets", + "data_key": "ets" }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - } - ], - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [ { - "values": [ - "tag1", - "tag2" - ], + "value": "tag1", "action": "remove" }, { - "values": [ - "tag3", - "tag4" - ], - "action": "add" + "value": "tag3", + "action": "upsert" } ] } @@ -361,44 +342,20 @@ export const TestInputsForDatasetUpdate = { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" }, { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" } ] } } }, - DATASET_UPDATE_WITH_SAME_TAGS_ADD: { - ...requestStructure, request: { - "dataset_id": "telemetry", - "version_key": validVersionKey, - "name": "sb-telemetry", - "tags": [ - { - "values": [ - "tag1", - "tag1" - ], - "action": "remove" - }, - { - "values": [ - "tag4", - "tag4" - ], - "action": "add" - } - ] - } - }, - DATASET_UPDATE_WITH_SAME_DENORM_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", @@ -407,16 +364,18 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" }, { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" } @@ -425,67 +384,32 @@ export const TestInputsForDatasetUpdate = { } }, + DATASET_UPDATE_WITH_EXISTING_DENORM: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "name": "sb-telemetry", + "denorm_config": { + "denorm_fields": [ + { + "value": { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }, + "action": "upsert" + } + ] + } + } + }, + DATASET_UPDATE_WITH_SAME_TRANSFORMATION_ADD_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, "name": "sb-telemetry", - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "add" - }, - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "add" - }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - } - ] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }] } } } diff --git a/api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts similarity index 78% rename from api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts rename to api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts index ee904424..06dccdaf 100644 --- a/api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts @@ -55,21 +55,21 @@ export const TestInputsForGenerateURL = { VALID_RESPONSE_FOR_MULTIFILES: [ { "filePath": `container/api-service/user-upload/telemetry.json`, - "fileName": 'telemetry.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC' + "fileName": "telemetry.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC" }, { - "filePath": 'container/api-service/user-upload/school-data.json', - "fileName": 'school-data.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/school-data.json?X-Amz-Algorithm=AWS4-HMAC' + "filePath": "container/api-service/user-upload/school-data.json", + "fileName": "school-data.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/school-data.json?X-Amz-Algorithm=AWS4-HMAC" } ], VALID_RESPONSE_FOR_SINGLE_FILE: [ { - "filePath": 'container/api-service/user-upload/telemetry.json', - "fileName": 'telemetry.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC' + "filePath": "container/api-service/user-upload/telemetry.json", + "fileName": "telemetry.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC" } ] } \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts similarity index 98% rename from api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts rename to api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts index 8d30ddb6..597b3b7c 100644 --- a/api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts +++ b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId, code } from "../../../controllers/GenerateSignedURL/GenerateSignedURL"; import { TestInputsForGenerateURL } from "./Fixtures"; diff --git a/api-service/src/v2/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts similarity index 98% rename from api-service/src/v2/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts index b7f60e27..4bedf75f 100644 --- a/api-service/src/v2/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { createTemplateFixtures } from "./Fixtures" import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.create' +const apiId = "api.query.template.create" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/v2/tests/QueryTemplates/CreateTemplate/Fixtures.ts b/api-service/src/tests/QueryTemplates/CreateTemplate/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/CreateTemplate/Fixtures.ts rename to api-service/src/tests/QueryTemplates/CreateTemplate/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts similarity index 94% rename from api-service/src/v2/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts index 4108415b..47377590 100644 --- a/api-service/src/v2/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts @@ -1,10 +1,10 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.delete' +const apiId = "api.query.template.delete" chai.use(spies); chai.should(); @@ -20,7 +20,7 @@ describe("DELETE QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "destroy", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1' + template_id: "sql1" } }) }) diff --git a/api-service/src/v2/tests/QueryTemplates/ListTemplates/Fixtures.ts b/api-service/src/tests/QueryTemplates/ListTemplates/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/ListTemplates/Fixtures.ts rename to api-service/src/tests/QueryTemplates/ListTemplates/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts similarity index 98% rename from api-service/src/v2/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts rename to api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts index e9f843d1..16800dcd 100644 --- a/api-service/src/v2/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts +++ b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; import { listTemplateFixtures } from "./Fixtures"; -const apiId = 'api.query.template.list' +const apiId = "api.query.template.list" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/v2/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts similarity index 86% rename from api-service/src/v2/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts index 1f6e129e..6a7078f2 100644 --- a/api-service/src/v2/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts @@ -1,10 +1,10 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.read' +const apiId = "api.query.template.read" chai.use(spies); chai.should(); @@ -20,12 +20,12 @@ describe("READ QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1', - template_name: 'sql1', - query: '"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} LIMIT 1"', - query_type: 'sql', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "sql1", + template_name: "sql1", + query: "\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} LIMIT 1\"", + query_type: "sql", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-29T11:29:58.759Z", updated_date: "2024-04-29T11:29:58.759Z" } diff --git a/api-service/src/v2/tests/QueryTemplates/TemplateQuerying/Fixtures.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/TemplateQuerying/Fixtures.ts rename to api-service/src/tests/QueryTemplates/TemplateQuerying/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts similarity index 82% rename from api-service/src/v2/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts rename to api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts index ff384181..8da1818b 100644 --- a/api-service/src/v2/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts +++ b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts @@ -1,14 +1,14 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; import { Datasource } from "../../../models/Datasource"; import nock from "nock"; import { config } from "../../../configs/Config"; import { templateQueryApiFixtures } from "./Fixtures"; -const apiId = 'api.query.template.query'; +const apiId = "api.query.template.query"; const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" chai.use(spies); @@ -34,12 +34,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1', - template_name: 'sql1', - query: '"SELECT * FROM {{DATASET}} WHERE \"__time\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}"', - query_type: 'sql', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "sql1", + template_name: "sql1", + query: "\"SELECT * FROM {{DATASET}} WHERE \"__time\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\"", + query_type: "sql", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04 - 30T05: 57:04.387Z", updated_date: "2024-04 - 30T05: 57:04.387Z" } @@ -77,12 +77,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'jsontemplate1', - template_name: 'jsontemplate1', - query: '{"queryType":"timeseries","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMIT}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]}', - query_type: 'json', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "jsontemplate1", + template_name: "jsontemplate1", + query: "{\"queryType\":\"timeseries\",\"datasetId\":\"{{DATASET}}\",\"intervals\":\"{{STARTDATE}}/{{ENDDATE}}\",\"limit\":\"{{LIMIT}}\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"school_id\"}},\"name\":\"school_id\"}]}", + query_type: "json", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-28T23:28:35.868Z", updated_date: "2024-04-28T23:28:35.868Z" } @@ -121,12 +121,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'jsontemplate1', - template_name: 'jsontemplate1', - query: '{"queryType":"timeseries","datasetId"::::"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMIT}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]}', - query_type: 'json', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "jsontemplate1", + template_name: "jsontemplate1", + query: "{\"queryType\":\"timeseries\",\"datasetId\"::::\"{{DATASET}}\",\"intervals\":\"{{STARTDATE}}/{{ENDDATE}}\",\"limit\":\"{{LIMIT}}\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"school_id\"}},\"name\":\"school_id\"}]}", + query_type: "json", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-28T23:28:35.868Z", updated_date: "2024-04-28T23:28:35.868Z" } diff --git a/api-service/src/v2/tests/QueryTemplates/UpdateTemplate/Fixtures.ts b/api-service/src/tests/QueryTemplates/UpdateTemplate/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/UpdateTemplate/Fixtures.ts rename to api-service/src/tests/QueryTemplates/UpdateTemplate/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts similarity index 98% rename from api-service/src/v2/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts index 195018c3..af50d77a 100644 --- a/api-service/src/v2/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { updateTemplateFixtures } from "./Fixtures" import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.update' +const apiId = "api.query.template.update" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/v2/tests/QueryWrapper/SqlWrapper/Fixtures.ts b/api-service/src/tests/QueryWrapper/SqlWrapper/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryWrapper/SqlWrapper/Fixtures.ts rename to api-service/src/tests/QueryWrapper/SqlWrapper/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts similarity index 95% rename from api-service/src/v2/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts rename to api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts index f389a394..510b8af4 100644 --- a/api-service/src/v2/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts +++ b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts @@ -1,12 +1,12 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForSqlWrapper } from "./Fixtures"; -import { druidHttpService } from "../../../controllers/QueryWrapper/SqlQueryWrapper"; import httpStatus from "http-status"; +import { druidHttpService } from "../../../connections/druidConnection"; const apiId = "api.obsrv.data.sql-query"; chai.use(spies); diff --git a/api-service/src/types/AlertModels.ts b/api-service/src/types/AlertModels.ts new file mode 100644 index 00000000..ae741db6 --- /dev/null +++ b/api-service/src/types/AlertModels.ts @@ -0,0 +1,36 @@ +type IPromiseAny = Promise; + +interface IAlert { + publishAlert(payload: Record): IPromiseAny; + getAlerts(payload: Record): IPromiseAny; + deleteAlert(payload: Record): IPromiseAny; +} + +interface INotificationChannel { + createNotificationChannel(payload: Record): IPromiseAny + testNotificationChannel(payload: Record, message: string): IPromiseAny; + updateNotificationChannel(payload: Record): IPromiseAny; +} + +interface IManager extends IAlert, INotificationChannel { + name: string +} + +export interface IGrafana extends IManager { + name: "grafana" +} + +export interface IPrometheus extends IManager { + name: "prometheus" +} + +interface IChannelService { + generateConfigPayload: (payload: Record) => Record, + testChannel: (payload: Record) => Promise +} + +export interface IChannelConfig { + name: string, + service: IChannelService +} + diff --git a/api-service/src/v2/types/ConfigModels.ts b/api-service/src/types/ConfigModels.ts similarity index 63% rename from api-service/src/v2/types/ConfigModels.ts rename to api-service/src/types/ConfigModels.ts index 538685aa..a03ec5f4 100644 --- a/api-service/src/v2/types/ConfigModels.ts +++ b/api-service/src/types/ConfigModels.ts @@ -1,3 +1,6 @@ +import { IngestionConfig } from "./IngestionModels"; +import { IDataSourceRules } from "./QueryModels"; + export interface ExtractionConfig { is_batch_event: boolean; extraction_key: string; @@ -41,4 +44,17 @@ export interface DatasetConfig { redis_db: number; index_data: boolean; } +export interface DataSetConfig { + querying: IDataSourceRules + indexConfiguration: IngestionConfig, + processing: DatasetProcessing +} +export interface DatasetProcessing { + topic: string; + extraction: ExtractionConfig; + dedup_config: DedupConfig; + validation_config: ValidationConfig; + denorm_config: DenormConfig; + router_config: RouterConfig; +} diff --git a/api-service/src/v1/models/ConnectionModels.ts b/api-service/src/types/ConnectionModels.ts similarity index 100% rename from api-service/src/v1/models/ConnectionModels.ts rename to api-service/src/types/ConnectionModels.ts diff --git a/api-service/src/v2/types/DatasetModels.ts b/api-service/src/types/DatasetModels.ts similarity index 81% rename from api-service/src/v2/types/DatasetModels.ts rename to api-service/src/types/DatasetModels.ts index 96e38af1..e67f902b 100644 --- a/api-service/src/v2/types/DatasetModels.ts +++ b/api-service/src/types/DatasetModels.ts @@ -45,7 +45,7 @@ export interface Result { } export enum DatasetStatus { - Live = "Live", Retired = "Retired", Draft = "Draft", ReadyToPublish = "ReadyToPublish" + Live = "Live", Retired = "Retired", Draft = "Draft", ReadyToPublish = "ReadyToPublish", Archived = "Archived" } export enum TransformationMode { @@ -57,5 +57,17 @@ export enum ValidationMode { } export enum DatasetType { - Dataset = "dataset", MasterDataset = "master-dataset" + event = "event", master = "master", transaction = "transaction" } + +export enum DatasetAction { + Retire = "Retire", Delete = "Delete", ReadyToPublish = "ReadyToPublish", Live="Live" +} + +export enum HealthStatus { + Healthy = "Healthy", UnHealthy= "UnHealthy" +} + +export enum DataSourceType { + druid = "druid" +} \ No newline at end of file diff --git a/api-service/src/v1/models/ExhaustModels.ts b/api-service/src/types/ExhaustModels.ts similarity index 100% rename from api-service/src/v1/models/ExhaustModels.ts rename to api-service/src/types/ExhaustModels.ts diff --git a/api-service/src/v2/types/IngestionModels.ts b/api-service/src/types/IngestionModels.ts similarity index 83% rename from api-service/src/v2/types/IngestionModels.ts rename to api-service/src/types/IngestionModels.ts index 9fc4237b..b43f0805 100644 --- a/api-service/src/v2/types/IngestionModels.ts +++ b/api-service/src/types/IngestionModels.ts @@ -50,3 +50,15 @@ export interface IngestionSchemeRequest { schema: Map[], config: IngestionConfig } + +export interface RollupInfo { + summary?: RollupSuggestionsSummary; +} + +interface RollupSuggestionsSummary { + [key: string]: { + path: string; + cardinality: number; + index: boolean; + }; +} \ No newline at end of file diff --git a/api-service/src/v2/types/MetricModel.ts b/api-service/src/types/MetricModel.ts similarity index 90% rename from api-service/src/v2/types/MetricModel.ts rename to api-service/src/types/MetricModel.ts index 7d63c191..ed68d0f9 100644 --- a/api-service/src/v2/types/MetricModel.ts +++ b/api-service/src/types/MetricModel.ts @@ -14,5 +14,5 @@ export interface Metric { } export enum Entity { - Data_in = "data-in", Data_out = "data-out", Management = "management" + Data_in = "data-in", Data_out = "data-out", Management = "management", DruidProxy = "druid-proxy" } \ No newline at end of file diff --git a/api-service/src/types/ObsrvError.ts b/api-service/src/types/ObsrvError.ts new file mode 100644 index 00000000..94fef6a2 --- /dev/null +++ b/api-service/src/types/ObsrvError.ts @@ -0,0 +1,24 @@ +export class ObsrvError { + code: string; + message: string; + errCode: string; + statusCode: number; + data: any | undefined; + datasetId: string; + err: Error | undefined; + + constructor(datasetId: string, code: string, message: string, errorCode: string, statusCode: number, err?: Error, data?: any) { + this.datasetId = datasetId; + this.code = code; + this.message = message; + this.errCode = errorCode; + this.statusCode = statusCode; + this.data = data; + this.err = err; + } + +} + +export const obsrvError = (datasetId: string, code: string, message: string, errorCode: string, statusCode: number, err?: Error, data?: any) => { + return new ObsrvError(datasetId, code, message, errorCode, statusCode, err, data) +} \ No newline at end of file diff --git a/api-service/src/v1/models/QueryModels.ts b/api-service/src/types/QueryModels.ts similarity index 100% rename from api-service/src/v1/models/QueryModels.ts rename to api-service/src/types/QueryModels.ts diff --git a/api-service/src/v2/types/ResponseModel.ts b/api-service/src/types/ResponseModel.ts similarity index 100% rename from api-service/src/v2/types/ResponseModel.ts rename to api-service/src/types/ResponseModel.ts diff --git a/api-service/src/v2/types/SampleURLModel.ts b/api-service/src/types/SampleURLModel.ts similarity index 100% rename from api-service/src/v2/types/SampleURLModel.ts rename to api-service/src/types/SampleURLModel.ts diff --git a/api-service/src/types/SchemaModel.ts b/api-service/src/types/SchemaModel.ts new file mode 100644 index 00000000..eb592d8a --- /dev/null +++ b/api-service/src/types/SchemaModel.ts @@ -0,0 +1,115 @@ +export interface DataSetConfig { + querying: Record + indexConfiguration: Record + processing: Record + } + + +export interface DatasetSchemeRequest { + data: Map[], + config: DatasetSchemaConfig +} + +export interface DatasetSchemaConfig { + dataset: string, + isBatch?: boolean + extractionKey: string, +} + +export interface DatasetSchemaResponse { + schema: any; + configurations: DataSetConfig + dataMappings: Record +} + +export interface DataSetConfig { + querying: Record + indexConfiguration: Record + processing: Record + } + + +export interface DatasetSchemaResponse { + schema: any; + configurations: DataSetConfig + dataMappings: Record +} + +export interface DatasetSchemeRequest { + data: Map[], + config: DatasetSchemaConfig +} + +export interface DatasetSchemaConfig { + dataset: string, + isBatch?: boolean + extractionKey: string, +} + +export interface SuggestionsTemplate { + property: string; + suggestions: Suggestion[]; +} + +export interface Suggestion { + message: string; + arrivalConflict?: boolean; + advice: string; + resolutionType: string; + severity: string; +} + +export interface ConflictTypes { + schema: Conflict; + required: Conflict; + formats: Conflict; + absolutePath: string; +} + +export interface Conflict { + property: string, + type: string, + path: string, + conflicts: any, + values: any[], + resolution: any, + severity: string +} + + +export interface FlattenSchema { + property: string + dataType: string + isRequired: boolean + path: string | any; + absolutePath: string + formate: string +} + +export interface Occurance { + property: { [key: string]: number }; + dataType: { [key: string]: number }; + isRequired: { [key: string]: number }; + path: { [key: string]: number }; + absolutePath: { [key: string]: number }; + format: { [key: string]: number }; +} + +export interface UniqueValues { + [key: string]: any[]; +} + +export interface FieldSchema { + type?: string | any; + format?: string + properties?: Record; + items?: FieldSchema; +} + +export interface RollupSummary { + [key: string]: { + path: string; + cardinality: number; + index: boolean; + }; +} diff --git a/api-service/src/v1/models/ValidationModels.ts b/api-service/src/types/ValidationModels.ts similarity index 100% rename from api-service/src/v1/models/ValidationModels.ts rename to api-service/src/types/ValidationModels.ts diff --git a/api-service/src/v2/utils/common.ts b/api-service/src/utils/common.ts similarity index 100% rename from api-service/src/v2/utils/common.ts rename to api-service/src/utils/common.ts diff --git a/api-service/src/v1/configs/Config.ts b/api-service/src/v1/configs/Config.ts deleted file mode 100644 index ca9834a1..00000000 --- a/api-service/src/v1/configs/Config.ts +++ /dev/null @@ -1,109 +0,0 @@ -// These configurations provide settings and values for various aspects of dataset management, data ingestion, and table configurations in a system. - -const env = process.env.system_env || "local" - -export const config = { - "env": env, - "api_port": process.env.api_port || 3000, - "body_parser_limit": process.env.body_parser_limit || "100mb", - "version": "1.0", - "query_api": { - "druid": { - "queryType": "realtime", - "host": process.env.druid_host || "http://localhost", - "port": process.env.druid_port || 8888, - "sql_query_path": "/druid/v2/sql/", - "native_query_path": "/druid/v2", - "list_datasources_path": "/druid/v2/datasources", - "submit_ingestion": "druid/indexer/v1/supervisor" - }, - "lakehouse": { - "queryType": "datalake", - "host": process.env.lakehouse_host || "http://localhost", - "port": process.env.lakehouse_port || 8080, - "catalog": process.env.lakehouse_catalog || "hudi_connector", - "schema": process.env.lakehouse_schema || "obsrv", - "default_user": process.env.lakehouse_default_user || "admin" - } - }, - "db_connector_config": { - client: "postgresql", - connection: { - host: process.env.postgres_host || 'localhost', - port: process.env.postgres_port || 5432, - database: process.env.postgres_database || 'obsrv', - user: process.env.postgres_username || 'postgres', - password: process.env.postgres_password || 'postgres', - } - }, - "telemetry_service_config": { - level: process.env.telemetry_log_level || 'info', - localStorageEnabled: process.env.telemetry_local_storage_enabled || 'true', - dispatcher: process.env.telemetry_local_storage_type || 'kafka', - telemetryProxyEnabled: process.env.telemetry_proxy_enabled, - proxyURL: process.env.telemetry_proxy_url, - proxyAuthKey: process.env.telemetry_proxy_auth_key, - compression_type: process.env.telemetry_kafka_compression || 'none', - filename: process.env.telemetry_file_filename || 'telemetry-%DATE%.log', - maxsize: process.env.telemetry_file_maxsize || 10485760, - maxFiles: process.env.telemetry_file_maxfiles || '100', - "kafka": { // The default Kafka configuration includes essential parameters such as broker IP addresses and other configuration options. - "config": { - "brokers": [`${process.env.kafka_host || 'localhost'}:${process.env.kafka_port || 9092}`], - "clientId": process.env.client_id || "obsrv-apis", - "retry": { - "initialRetryTime": process.env.kafka_initial_retry_time ? parseInt(process.env.kafka_initial_retry_time) : 3000, - "retries": process.env.kafka_retries ? parseInt(process.env.kafka_retries) : 1 - }, - "connectionTimeout": process.env.kafka_connection_timeout ? parseInt(process.env.kafka_connection_timeout) : 5000 - }, - "topics": { // Default Kafka topics depend on type of dataset. - "createDataset": `${process.env.system_env || 'local'}.ingest`, - "createMasterDataset": `${process.env.system_env || 'local'}.masterdata.ingest` - } - } - }, - "dataset_types": { - normalDataset: "dataset", - masterDataset: "master-dataset" - }, - "datasource_storage_types": { - druid: "druid", - datalake: "datalake" - }, - "redis_config": { - "redis_host": process.env.redis_host || 'localhost', - "redis_port": process.env.redis_port || 6379 - }, - "exclude_datasource_validation": process.env.exclude_datasource_validation ? process.env.exclude_datasource_validation.split(",") : ["system-stats", "failed-events-summary", "masterdata-system-stats", "system-events"], // list of datasource names to skip validation while calling query API - "telemetry_dataset": process.env.telemetry_dataset || `${env}.system.telemetry.events`, - "table_names": { // Names of all tables available for CRUD operations - "datasets": "datasets", - "datasources": "datasources", - "datasetSourceConfig": "dataset_source_config" - }, - "table_config": { // This object defines the configuration for each table. - "datasets": { - "primary_key": "id", - "references": [] - }, - "datasources": { - "primary_key": "id", - "references": [] - }, - "dataset_source_config": { - "primary_key": "id", - "references": [] - } - }, - "exhaust_config": { - "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure - "cloud_storage_region": process.env.cloud_storage_region || "", // Region for the cloud provider storage - "cloud_storage_config": process.env.cloud_storage_config ? JSON.parse(process.env.cloud_storage_config) : {}, // Respective credentials object for cloud provider. Optional if service account provided - "container": process.env.container || "", // Storage container/bucket name - "container_prefix": process.env.container_prefix || "", // Path to the folder inside container/bucket. Empty if data at root level - "storage_url_expiry": process.env.storage_url_expiry ? parseInt(process.env.storage_url_expiry) : 3600, // in seconds, Default 1hr of expiry for Signed URLs. - "maxQueryDateRange": process.env.exhaust_query_range ? parseInt(process.env.exhaust_query_range) : 31, // in days. Defines the maximum no. of days the files can be fetched - "exclude_exhaust_types": process.env.exclude_exhaust_types ? process.env.exclude_exhaust_types.split(",") : ["system-stats", "masterdata-system-stats", "system-events",] // list of folder type names to skip exhaust service - }, -} diff --git a/api-service/src/v1/configs/Extensions.ts b/api-service/src/v1/configs/Extensions.ts deleted file mode 100644 index 967e0e09..00000000 --- a/api-service/src/v1/configs/Extensions.ts +++ /dev/null @@ -1,11 +0,0 @@ - -export const extensions: Array = [ - - -] - - -interface IExtensionConfig { - id: string; - routePath: string; -} \ No newline at end of file diff --git a/api-service/src/v1/configs/RoutesConfig.ts b/api-service/src/v1/configs/RoutesConfig.ts deleted file mode 100644 index 37720b2b..00000000 --- a/api-service/src/v1/configs/RoutesConfig.ts +++ /dev/null @@ -1,181 +0,0 @@ -export const routesConfig = { - default: { - api_id: "api", - validation_schema: null, - }, - query: { - native_query: { - api_id: "native.query", - method: "post", - path: "/data/v1/query", - validation_schema: "QueryRequest.json", - }, - native_query_with_params: { - api_id: "native.query", - method: "post", - path: "/data/v1/query/:datasetId", - validation_schema: "QueryRequest.json", - }, - sql_query: { - api_id: "sql.query", - method: "post", - path: "/data/v1/sql-query", - validation_schema: "QueryRequest.json", - }, - sql_query_with_params: { - api_id: "sql.query", - method: "post", - path: "/data/v1/sql-query/:datasetId", - validation_schema: "QueryRequest.json", - }, - }, - config: { - dataset: { - save: { - api_id: "config.dataset.create", - method: "post", - path: "/datasets/v1/create", - validation_schema: "DatasetCreateReq.json", - }, - read: { - api_id: "config.dataset.read", - method: "get", - path: "/datasets/v1/get/:datasetId", - validation_schema: null, - }, - update: { - api_id: "config.dataset.update", - method: "patch", - path: "/datasets/v1/update", - validation_schema: "DatasetUpdateReq.json", - }, - list: { - api_id: "config.dataset.list", - method: "post", - path: "/datasets/v1/list", - validation_schema: "DatasetListReq.json", - }, - }, - datasource: { - save: { - api_id: "config.datasource.create", - method: "post", - path: "/datasources/v1/create", - validation_schema: "DatasourceSaveReq.json", - }, - read: { - api_id: "config.datasource.read", - method: "get", - path: "/datasources/v1/get/:datasourceId", - validation_schema: null, - }, - update: { - api_id: "config.datasource.update", - method: "patch", - path: "/datasources/v1/update", - validation_schema: "DatasourceUpdateReq.json", - }, - list: { - api_id: "config.datasource.list", - method: "post", - path: "/datasources/v1/list", - validation_schema: "DatasetListReq.json", - }, - }, - dataset_source_config: { - save: { - api_id: "config.dataset.source.config.create", - method: "post", - path: "/datasets/v1/source/config/create", - validation_schema: "DatasetSourceConfigSaveReq.json", - }, - read: { - api_id: "config.dataset.source.config.read", - method: "get", - path: "/datasets/v1/source/config/get/:datasetId", - validation_schema: null, - }, - update: { - api_id: "config.dataset.source.config.update", - method: "patch", - path: "/datasets/v1/source/config/update", - validation_schema: "DatasetSourceConfigUpdateReq.json", - }, - list: { - api_id: "config.dataset.source.config.list", - method: "post", - path: "/datasets/v1/source/config/list", - validation_schema: "DatasetListReq.json", - }, - } - - }, - data_ingest: { - api_id: "dataset.data.in", - method: "post", - path: "/data/v1/in/:datasetId", - validation_schema: "DataIngestionReq.json", - }, - tenant_ingest: { - api_id: "dataset.data.in", - method: "post", - path: "/data/tenant/in/:datasetId", - validation_schema: "DataIngestionReq.json", - }, - exhaust: { - api_id: "dataset.data.exhaust", - method: "get", - path: "/data/v1/exhaust/:datasetId", - validation_schema: "DataExhaustReq.json" - }, - prometheus: { - method: "get", - path: "/metrics", - validation_schema: null, - }, - submit_ingestion: { - api_id: "submit.ingestion", - method: "post", - path: "/data/v1/submit/ingestion", - validation_schema: "SubmitIngestionReq.json" - }, - query_wrapper: { - sql_wrapper: { - api_id: "query.wrapper.sql.query", - method: "post", - path: "/v1/sql", - }, - native_post: { - api_id: "query.wrapper.native.post", - method: "post", - path: /\/druid\/v2.*/, - }, - native_get: { - api_id: "query.wrapper.native.get", - method: "get", - path: /\/druid\/v2.*/ - }, - native_delete: { - api_id: "query.wrapper.native.delete", - method: "delete", - path: "/druid/v2/:queryId" - }, - druid_status: { - api_id: "query.wrapper.status", - method: "get", - path: "/status" - } - }, - health: { - api_id: "api.health", - method: "get", - path: "/health" - }, - schema_validator:{ - api_id: "api.schema.validate", - method: "post", - path: "/data/v1/schema/validate", - validation_schema: "SchemaValidatorReq.json" - } -} - diff --git a/api-service/src/v1/connectors/DbConnector.ts b/api-service/src/v1/connectors/DbConnector.ts deleted file mode 100644 index f3b4f067..00000000 --- a/api-service/src/v1/connectors/DbConnector.ts +++ /dev/null @@ -1,179 +0,0 @@ -import knex, { Knex } from "knex"; -import { IConnector } from "../models/DatasetModels"; -import { DbConnectorConfig } from "../models/ConnectionModels"; -import { SchemaMerger } from "../generators/SchemaMerger"; -import constants from '../resources/Constants.json' -import { config as appConfig } from "../configs/Config" -import _ from 'lodash' -import { wrapperService } from "../routes/Router"; -const schemaMerger = new SchemaMerger() - -const OP_TYPES = { - INSERT: "insert", - UPDATE: "update", - UPSERT: "upsert", - READ: "read", - LIST: "list", - DELETE: "delete", - SQL: "sql", -} -export class DbConnector implements IConnector { - public pool: Knex - private config: DbConnectorConfig - public typeToMethod = { - insert: this.insertRecord, - update: this.updateRecord, - read: this.readRecords, - upsert: this.upsertRecord, - } - public method: any - constructor(config: DbConnectorConfig) { - this.config = config; - this.pool = knex(this.config) - } - public init = () => { - this.connect() - .then(() => console.info("Database Connection Established...")) - .catch((err: Error) => console.error(`Database Connection failed: ${err.message}`)) - } - - async connect() { - await this.pool.select(1) - } - - async close() { - return await this.pool.destroy() - } - - async health() { - await this.connect() - await this.pool.select(1) - } - - execute(type: keyof typeof this.typeToMethod, property: any) { - this.method = this.typeToMethod[ type ] - return this.method(property[ "table" ], property[ "fields" ]) - } - - public async insertRecord(table: string, fields: any) { - await this.pool.transaction(async (dbTransaction) => { - await this.submit_ingestion(_.get(fields, 'ingestion_spec'), table, _.get(fields, "type")) - await dbTransaction(table).insert(fields).on('query-error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, fields); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, fields); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }); - }) - } - - public async updateRecord(table: string, fields: any) { - const { filters, values } = fields - await this.pool.transaction(async (dbTransaction) => { - const currentRecord = await dbTransaction(table).select(Object.keys(values)).where(filters).first() - if (_.isUndefined(currentRecord)) { throw constants.FAILED_RECORD_UPDATE } - if (!_.isUndefined(currentRecord.tags)) { delete currentRecord.tags } - await dbTransaction(table).where(filters).update(schemaMerger.mergeSchema(currentRecord, values)).on('query-error', (error: any) => { - this.log_error(OP_TYPES.UPDATE, error, table, values); - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, values); - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }); - }) - } - - public async upsertRecord(table: string, fields: any) { - const { filters, values } = fields; - const existingRecord = await this.pool(table).select().where(filters).first() - if (!_.isUndefined(existingRecord)) { - await this.pool.transaction(async (dbTransaction) => { - await this.submit_ingestion(_.get(values, 'ingestion_spec'), table, _.get(fields, "type")) - await dbTransaction(table).where(filters).update(schemaMerger.mergeSchema(existingRecord, values)).on('query-error', (error: any) => { - this.log_error(OP_TYPES.UPSERT, error, table, values); - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, values); - throw constants.FAILED_RECORD_CREATE - }); - }) - } else { - await this.pool.transaction(async (dbTransaction) => { - await this.submit_ingestion(_.get(values, 'ingestion_spec'), table, _.get(fields, "type")) - await dbTransaction(table).insert(values).on('query-error', (error: any) => { - this.log_error(OP_TYPES.UPSERT, error, table, values); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, values); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }); - }) - } - } - - public async readRecords(table: string, fields: any) { - const query = this.pool.from(table).select().where((builder) => { - const filters = fields.filters || {}; - if (filters.status) { - if (Array.isArray(filters.status) && filters.status.length > 0) { - builder.whereIn("status", filters.status); - } else if (filters.status.length > 0) { - builder.where("status", filters.status); - } - } - delete filters.status; - builder.where(filters).on('query-error', (error: any) => { - this.log_error(OP_TYPES.READ, error, table, filters); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, filters); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }); - }) - return await query - } - - public async listRecords(table: string) { - return await this.pool.select('*').from(table).on('query-error', (error: any) => { - this.log_error(OP_TYPES.LIST, error, table, {}); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, {}); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }) - } - - private async submit_ingestion(ingestion_spec: Record, table: string, storage_type: string) { - if (appConfig.table_names.datasources === table && storage_type === appConfig.datasource_storage_types.druid) { - return await wrapperService.submitIngestion(ingestion_spec) - .catch((error: any) => { - console.error(constants.INGESTION_FAILED_ON_SAVE) - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }) - } - return - } - - public async executeSql(sql: string[]) { - let result: any[] = []; - await this.pool.transaction(async (dbTransaction) => { - for(let query of sql) { - result.push(await dbTransaction.raw(query).on('query-error', (error: any) => { - this.log_error(OP_TYPES.SQL, error, JSON.stringify(query), {}); - throw {...constants.FAILED_SQL_QUERY, "errCode": error.code}; - })); - } - }); - return result; - } - - private log_error(op_type: string, error: any, table: string, values: any) { - console.log(` - Error occured for operation ${op_type} - - Table - ${table} - Values - ${JSON.stringify(values)} - Error - ${JSON.stringify(error)} - `); - } -} diff --git a/api-service/src/v1/connectors/HttpConnector.ts b/api-service/src/v1/connectors/HttpConnector.ts deleted file mode 100644 index 26eff56a..00000000 --- a/api-service/src/v1/connectors/HttpConnector.ts +++ /dev/null @@ -1,28 +0,0 @@ -import axios, { AxiosInstance } from "axios"; -import { IConnector } from "../models/DatasetModels"; -export class HTTPConnector implements IConnector { - private url: string; - constructor(url: string) { - this.url = url - } - connect(): AxiosInstance { - return axios.create({ - baseURL: this.url, - timeout: 3000, - headers: { "Content-Type": "application/json" } - }); - } - - execute(sample: string) { - throw new Error("Method not implemented."); - } - executeSql(sql: string[]) { - throw new Error("Method not implemented."); - } - close() { - throw new Error("Method not implemented."); - } - -} - - diff --git a/api-service/src/v1/connectors/KafkaConnector.ts b/api-service/src/v1/connectors/KafkaConnector.ts deleted file mode 100644 index 3ce6f848..00000000 --- a/api-service/src/v1/connectors/KafkaConnector.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IConnector } from "../models/DatasetModels" -const telemetryService = require('../lib/services/TelemetryService') - -export class KafkaConnector implements IConnector { - public telemetryService: any - constructor() { - this.telemetryService = telemetryService - } - async connect() { - await telemetryService.health() - - } - - async execute(req: any, res: any, topic: any) { - await this.telemetryService.dispatch(req, res, topic) - } - - async executeSql(sql: string[]) { - throw new Error("Method not implemented") - } - - close() { - throw new Error("Method not implemented") - } -} diff --git a/api-service/src/v1/generators/SchemaMerger.ts b/api-service/src/v1/generators/SchemaMerger.ts deleted file mode 100644 index 942a1f9e..00000000 --- a/api-service/src/v1/generators/SchemaMerger.ts +++ /dev/null @@ -1,7 +0,0 @@ -import * as _ from "lodash"; -export class SchemaMerger { - - mergeSchema(schema: any, schema2: any): Map { - return _.merge(schema, schema2); - } -} diff --git a/api-service/src/v1/helpers/DatasetConfigs.ts b/api-service/src/v1/helpers/DatasetConfigs.ts deleted file mode 100644 index 3f620619..00000000 --- a/api-service/src/v1/helpers/DatasetConfigs.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { dbConnector } from "../routes/Router" -import { globalCache } from "../routes/Router"; - - -function getDatasetConfigs() { - return dbConnector.listRecords('datasets') -} - -export const refreshDatasetConfigs = async () => { - try { - const data = await getDatasetConfigs(); - globalCache.set("dataset-config", data) - } - catch (err) { - console.log(err); - throw new Error("Failed to cache configs") - } -} - diff --git a/api-service/src/v1/helpers/DatasetSourceConfigs.ts b/api-service/src/v1/helpers/DatasetSourceConfigs.ts deleted file mode 100644 index af3e0832..00000000 --- a/api-service/src/v1/helpers/DatasetSourceConfigs.ts +++ /dev/null @@ -1,52 +0,0 @@ -import _ from "lodash"; -import { SchemaMerger } from "../generators/SchemaMerger" -import { defaultConfig } from '../resources/schemas/DatasetConfigDefault' -import {v4} from 'uuid' -import { DatasetStatus } from "../models/DatasetModels"; -const schemaMerger = new SchemaMerger() -export class DatasetSourceConfigs { - private id: string - private dataset_id: string - private connector_type: string - private connector_config: object - private status: DatasetStatus - private connector_stats: object - private created_by: string - private updated_by: string - private published_date: Date - constructor(payload: any) { - if (payload.id) { - this.id = payload.id - } - else { - this.id = v4() - } - this.dataset_id = payload.dataset_id - this.connector_type = payload.connector_type - this.connector_config = payload.connector_config - this.status = payload.status - this.connector_stats = payload.connector_stats - this.created_by = payload.created_by - this.updated_by = payload.updated_by - this.published_date = payload.published_date - } - - public getValues() { - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, connector_type: this.connector_type, connector_config: this.connector_config, status: this.status, connector_stats: this.connector_stats, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date}), { "updated_date": new Date }) - } - - public setValues() { - return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()) - } - - public removeNullValues(payload: any) { - Object.keys(payload).map((value) => { - if (_.isEmpty(payload[value])) delete payload[value] - }) - return payload - } - - public getDefaults() { - return {...defaultConfig.sourceConfig} - } -} diff --git a/api-service/src/v1/helpers/Datasets.ts b/api-service/src/v1/helpers/Datasets.ts deleted file mode 100644 index 94a12771..00000000 --- a/api-service/src/v1/helpers/Datasets.ts +++ /dev/null @@ -1,93 +0,0 @@ -import _ from 'lodash' -import { ValidationConfig, ExtractionConfig, DedupConfig, DenormConfig, RouterConfig, DatasetConfig } from '../models/ConfigModels' -import { defaultConfig } from '../resources/schemas/DatasetConfigDefault' -import { SchemaMerger } from '../generators/SchemaMerger' -import { config } from '../configs/Config' -import { DatasetStatus } from '../models/DatasetModels' -import constants from "../resources/Constants.json"; - -let schemaMerger = new SchemaMerger() -export class Datasets { - private id: string - private type: string - private name: string - private dataset_id: string - private validation_config: object - private extraction_config: object - private dedup_config: object - private data_schema: object - private router_config: object - private denorm_config: object - private dataset_config: object - private tags: any - private status: DatasetStatus - private created_by: string - private updated_by: string - private published_date: Date - constructor(payload: any) { - if (payload.id) { - this.id = payload.id - } - else { - this.id = payload.dataset_id - } - this.dataset_id = payload.dataset_id - this.type = payload.type - this.name = payload.name - this.validation_config = payload.validation_config - this.extraction_config = payload.extraction_config - this.dedup_config = payload.dedup_config - this.data_schema = payload.data_schema - this.router_config = payload.router_config - this.denorm_config = payload.denorm_config - this.dataset_config = payload.dataset_config - this.tags = payload.tags - this.status = payload.status - this.created_by = payload.created_by - this.updated_by = payload.updated_by - this.published_date = payload.published_date - } - - public getValues() { - this.validateDenormConfig(); - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, type: this.type, name: this.name, validation_config: this.validation_config, extraction_config: this.extraction_config, dedup_config: this.dedup_config, data_schema: this.data_schema, router_config: this.router_config, denorm_config: this.denorm_config, dataset_config: this.dataset_config, tags: this.tags, status: this.status, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date }), { "updated_date": new Date }) - } - - public setValues() { - this.validateDenormConfig(); - return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()) - } - - public removeNullValues(payload: any) { - Object.keys(payload).map((value) => { - if (_.isEmpty(payload[value])) delete payload[value] - }) - return payload - } - - public getDefaults() { - if (this.type == config.dataset_types.masterDataset) { - return {...defaultConfig.master} - } - else { - return {...defaultConfig.dataset} - } - } - - private validateDenormConfig() { - if (this.denorm_config && _.has(this.denorm_config, 'denorm_fields')) { - let duplicatesExist = false; - let denormFields: any = _.get(this.denorm_config, 'denorm_fields', []); - denormFields = _.map(denormFields, (denormField: Record) => _.get(denormField, 'denorm_out_field')); - denormFields.map( - (denormField: string | number) => { - if(_.indexOf(denormFields, denormField) !== _.lastIndexOf(denormFields, denormField)) - duplicatesExist = true; - } - ); - if(duplicatesExist) { - throw constants.DUPLICATE_DENORM_FIELD; - } - } - } -} diff --git a/api-service/src/v1/helpers/Datasources.ts b/api-service/src/v1/helpers/Datasources.ts deleted file mode 100644 index 2a3d1051..00000000 --- a/api-service/src/v1/helpers/Datasources.ts +++ /dev/null @@ -1,65 +0,0 @@ -import _ from 'lodash' -import configDefault from '../resources/schemas/DatasourceConfigDefault.json' -import { SchemaMerger } from '../generators/SchemaMerger' -import { DatasetStatus } from '../models/DatasetModels' -let schemaMerger = new SchemaMerger - -export class Datasources { - private id: string - private dataset_id: string - private ingestion_spec: object - private type: string - private datasource: string - private datasource_ref: string - private retention_period: object - private archival_policy: object - private purge_policy: object - private backup_config: object - private status: DatasetStatus - private created_by: string - private updated_by: string - private version: string - private published_date: Date - private metadata: object - - constructor(payload: any) { - if (payload.id) { - this.id = payload.id - } - else { - this.id = payload.dataset_id + '_' + payload.datasource - } - this.dataset_id = payload.dataset_id - this.ingestion_spec = payload.ingestion_spec - this.type = payload.type - this.datasource = payload.datasource - this.datasource_ref = payload.datasource_ref - this.retention_period = payload.retentionPeriod - this.archival_policy = payload.archivalPolicy - this.purge_policy = payload.purgePolicy - this.backup_config = payload.backup_config - this.status = payload.status - this.version = payload.version - this.created_by = payload.created_by - this.updated_by = payload.updated_by - this.published_date = payload.published_date - this.metadata = payload.metadata - } - public getValues() { - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, ingestion_spec: this.ingestion_spec, type: this.type, datasource: this.datasource, datasource_ref: this.datasource_ref, retention_period: this.retention_period, archival_policy: this.archival_policy, purge_policy: this.purge_policy, backup_config: this.backup_config, status: this.status, version: this.version, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date, metadata: this.metadata }), { "updated_date": new Date }) - } - - public setValues() { - return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()); - } - - public removeNullValues(payload: any) { - Object.keys(payload).map((value) => { - if (_.isEmpty(payload[value])) delete payload[value] - }) - return payload - } - public getDefaults() { - return {...configDefault} - } -} diff --git a/api-service/src/v1/helpers/DbUtil.ts b/api-service/src/v1/helpers/DbUtil.ts deleted file mode 100644 index f9c95091..00000000 --- a/api-service/src/v1/helpers/DbUtil.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import constants from "../resources/Constants.json"; -import { config } from "../configs/Config"; -import _ from "lodash"; -import { IConnector } from "../models/DatasetModels"; - -export class DbUtil { - private dbConnector: IConnector - private table: string - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - } - public save = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - const fetchedRecord = await this.dbConnector.execute("read", { table: this.table, fields: { "filters": filters } }) - if (!_.isEmpty(fetchedRecord)) { throw constants.DUPLICATE_RECORD } - await this.dbConnector.execute("insert", { "table": this.table, "fields": fields }) - .then(() => { - console.log(constants.RECORD_SAVED) - ResponseHandler.successResponse(req, res, { status: 200, data: Object.assign({ message: constants.RECORD_SAVED }, filters) }); - }) - } - public update = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - await this.dbConnector.execute("update", { "table": this.table, "fields": { "filters": filters, "values": fields } }) - .then(() => { - console.log(constants.RECORD_UPDATED) - ResponseHandler.successResponse(req, res, { status: 200, data: Object.assign({ message: constants.RECORD_UPDATED }, filters) }); - }) - } - public upsert = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - await this.dbConnector.execute("upsert", { "table": this.table, "fields": { "filters": filters, "values": fields } }) - .then(() => { - console.log(constants.RECORD_UPDATED) - ResponseHandler.successResponse(req, res, { status: 200, data: Object.assign({ message: constants.RECORD_UPDATED }, filters) }); - }) - } - public read = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - await this.dbConnector.execute("read", { "table": this.table, "fields": { "filters": filters } }) - .then((data: any[]) => { - !_.isEmpty(data) ? ResponseHandler.successResponse(req, res, { status: 200, data: _.first(data) }) : (() => { throw constants.RECORD_NOT_FOUND; })() - }) - } - public list = async (req: Request, res: Response, next: NextFunction, fields: any) => { - await this.dbConnector.execute("read", { "table": this.table, "fields": fields }) - .then((data: any) => { - ResponseHandler.successResponse(req, res, { status: 200, data: data }); - }) - } - private getFilters = (table: string, fields: any) => { - let filters: any = {} - const table_config: any = config["table_config"][table as keyof typeof config.table_config] - filters[table_config["primary_key"]] = fields[table_config["primary_key"]] - return filters - } -} diff --git a/api-service/src/v1/helpers/ErrorResponseHandler.ts b/api-service/src/v1/helpers/ErrorResponseHandler.ts deleted file mode 100644 index f680c3e2..00000000 --- a/api-service/src/v1/helpers/ErrorResponseHandler.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { NextFunction, Request, Response } from "express"; -import httpStatus from "http-status"; -import { setAuditState } from "../services/telemetry"; - -export class ErrorResponseHandler { - private serviceName: string; - constructor(serviceName: string) { - this.serviceName = serviceName; - } - public handleError(req: Request, res: Response, next: NextFunction, error: any, audit: boolean = true): any { - console.error("Error in " + this.serviceName) - console.error(JSON.stringify({ - "ts": Date.now(), - "body": req.body, - "headers": req.headers, - "url": req.url, - "error": { - "message": error?.message, - "stack": error?.stack, - "data": error?.data, - "code": error?.code, - "error": error, - } - })); - if(audit) setAuditState("failed", req); - next({ - statusCode: error.status || httpStatus.INTERNAL_SERVER_ERROR, - message: error.message, - errCode: httpStatus[`${error.status}_NAME`] || httpStatus["500_NAME"], - }); - } -}; diff --git a/api-service/src/v1/helpers/LakehouseUtil.ts b/api-service/src/v1/helpers/LakehouseUtil.ts deleted file mode 100644 index cb5e2026..00000000 --- a/api-service/src/v1/helpers/LakehouseUtil.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Trino, BasicAuth } from 'trino-client'; -import _ from 'lodash'; -import { config } from '../configs/Config'; - -const trino: Trino = Trino.create({ - server: `${config.query_api.lakehouse.host}:${config.query_api.lakehouse.port}`, - catalog: config.query_api.lakehouse.catalog, - schema: config.query_api.lakehouse.schema, - auth: new BasicAuth(config.query_api.lakehouse.default_user), -}); - - -const getFormattedData = (data: any[], columnData: any[]) => { - const formattedData: any[] = []; - for (let i = 0; i < data.length; i++) { - const row = data[ i ]; - const jsonRow: any = {}; - for (let j = 0; j < row.length; j++) { - // assign column only if doesn't start with _hoodie_ - const colName = columnData[ j ]; - if (_.startsWith(colName, "_hoodie_")) { - continue; - } - jsonRow[ colName ] = row[ j ]; - } - formattedData.push(jsonRow); - } - return formattedData; -} - - -export const executeLakehouseQuery = async (query: string) => { - const iter = await trino.query(query); - let queryResult: any = [] - for await (let data of iter) { - if(!_.isEmpty(data.error)){ - throw { - status: 400, - message: data.error.message.replace(/line.*: /, ''), - code: "BAD_REQUEST" - } - } - queryResult = [ ...queryResult, ...(data?.data?.length ? data.data : []) ] - } - let columns = await iter.map((r: any) => r.columns ?? []).next(); - let finalColumns = columns.value.map((column: any) => { - return column.name; - }); - const formattedData = getFormattedData(queryResult, finalColumns); - return formattedData -} \ No newline at end of file diff --git a/api-service/src/v1/helpers/ResponseHandler.ts b/api-service/src/v1/helpers/ResponseHandler.ts deleted file mode 100644 index e2d35471..00000000 --- a/api-service/src/v1/helpers/ResponseHandler.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { ErrorRequestHandler, NextFunction, Request, Response } from "express"; -import httpStatus from "http-status"; -import { IResponse, Result } from "../models/DatasetModels"; -import constants from "../resources/Constants.json"; -import { routesConfig } from "../configs/RoutesConfig"; -import { onFailure, onSuccess } from "./../../v2/metrics/prometheus/helpers"; -type extendedErrorRequestHandler = ErrorRequestHandler & { - statusCode: number; - message: string; - errCode: string; - id?: string; -}; - -const ResponseHandler = { - successResponse: (req: Request, res: Response, result: Result) => { - const { entity } = req as any; - res.status(result.status || 200).json(ResponseHandler.refactorResponse({ id: (req as any).id, result: result.data })); - entity && onSuccess(req, res) - }, - - routeNotFound: (req: Request, res: Response, next: NextFunction) => { - next({ statusCode: httpStatus.NOT_FOUND, message: constants.ERROR_MESSAGE.ROUTE_NOT_FOUND, errCode: httpStatus["404_NAME"] }); - }, - - refactorResponse: ({ id = routesConfig.default.api_id, ver = "v1", params = { status: constants.STATUS.SUCCESS, errmsg: "" }, responseCode = httpStatus["200_NAME"], result = {} }): IResponse => { - return { id, ver, ts: Date.now(), params, responseCode, result } - }, - - errorResponse: (error: extendedErrorRequestHandler, req: Request, res: Response, next: NextFunction) => { - const { statusCode, message, errCode } = error; - const { id, entity } = req as any; - res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json(ResponseHandler.refactorResponse({ id: id, params: { status: constants.STATUS.FAILURE, errmsg: message, }, responseCode: errCode || httpStatus["500_NAME"] })); - entity && onFailure(req, res) - }, - - setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { - (req as any).id = id; - next(); - }, - - flatResponse: (req: Request, res: Response, result: Result) => { - const { entity } = req as any; - entity && onSuccess(req, res) - res.status(result.status).send(result.data); - }, -} - -export { ResponseHandler }; diff --git a/api-service/src/v1/helpers/ValidationService.ts b/api-service/src/v1/helpers/ValidationService.ts deleted file mode 100644 index 57d6d35f..00000000 --- a/api-service/src/v1/helpers/ValidationService.ts +++ /dev/null @@ -1,12 +0,0 @@ -import Ajv from "ajv"; -const validator = new Ajv({ strict: false }); - -export const schemaValidation = (payload: Record, schema: Record): Record => { - const isValid = validator.validate(schema, payload) - if (!isValid) { - const error: any = validator.errors; - const errorMessage = error[0]?.schemaPath?.replace("/", "") + " " + error[0]?.message || "Invalid Request Body"; - return { isValid, message: errorMessage } - } - return { isValid, message: "success" } -} \ No newline at end of file diff --git a/api-service/src/v1/lib/client-cloud-services/AWSStorageService.js b/api-service/src/v1/lib/client-cloud-services/AWSStorageService.js deleted file mode 100644 index d8661ec4..00000000 --- a/api-service/src/v1/lib/client-cloud-services/AWSStorageService.js +++ /dev/null @@ -1,475 +0,0 @@ -/** - * @file - AWS Storage Service - * @exports - `AWSStorageService` - * @since - 5.0.1 - * @version - 1.0.0 - * @implements - BaseStorageService - * @see {@link https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html | X-Amz-Credential} - * @see {@link https://docs.aws.amazon.com/directconnect/latest/APIReference/CommonParameters.html#CommonParameters-X-Amz-Credential | X-Amz-Credential} - */ - -const BaseStorageService = require("./BaseStorageService"); -const { logger } = require("@project-sunbird/logger"); -const _ = require("lodash"); -const dateFormat = require("dateformat"); -const uuidv1 = require("uuid/v1"); -const async = require("async"); -const storageLogger = require("./storageLogger"); -const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); -const { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, ListObjectsV2Command, } = require("@aws-sdk/client-s3"); -const { fromInstanceMetadata } = require("@aws-sdk/credential-providers"); -const { Upload } = require("@aws-sdk/lib-storage"); -const multiparty = require("multiparty"); -const moment = require("moment"); -const { config: globalConfig } = require("../../configs/Config"); -const { getFileKey } = require("../../utils/common"); - -class AWSStorageService extends BaseStorageService { - constructor(config) { - super(); - if (_.get(config, "identity") && _.get(config, "credential") && _.get(config, "region")) { - process.env.AWS_ACCESS_KEY_ID = _.get(config, "identity"); - process.env.AWS_SECRET_ACCESS_KEY = _.get(config, "credential"); - const region = _.get(config, "region").toString(); - this.client = new S3Client({ region }); - } else { - const region = globalConfig.exhaust_config.exhaust_region || "us-east-2"; - process.env.AWS_REGION = region; - const s3Client = new S3Client({ - region, - }); - this.client = s3Client; - } - } - - /** - * @description - Function to generate AWS command for an operation - * @param {string} bucketName - AWS bucket name - * @param {string} fileToGet - AWS File to fetch - * @param {string} prefix - `Optional` - Prefix for file path - * @returns - AWS Command to be executed by SDK - */ - getAWSCommand(bucketName, fileToGet, prefix = "") { - return new GetObjectCommand({ Bucket: bucketName, Key: prefix + fileToGet }); - } - - getAWSPutCommand(bucketName, fileToGet, prefix = "") { - return new PutObjectCommand({ Bucket: bucketName, Key: prefix + fileToGet }); - } - - listAWSCommand(bucketName, prefix = "",) { - return new ListObjectsV2Command({ Bucket: bucketName, Prefix: prefix, Delimiter: "/",}); - } - - /** - * @description - Function to check whether file exists in specified bucket or not - * @param {string} bucketName - AWS bucket name - * @param {string} fileToGet - AWS File to check - * @param {string} prefix - `Optional` - Prefix for file path - * @param {function} cb - Callback function - */ - async fileExists(bucketName, fileToGet, prefix = "", cb) { - const params = { Bucket: bucketName, Key: prefix + fileToGet }; - const command = new HeadObjectCommand(params); - logger.info({ msg: "AWS__StorageService - fileExists called for bucketName " + bucketName + " for file " + params.Key }); - await this.client - .send(command) - .then((resp) => { - cb(null, resp); - }) - .catch((err) => { - cb(err); - }); - } - - /** - * @description - Provides a stream to read from a storage - * @param {string} bucketName - Bucket name or folder name in storage service - * @param {string} fileToGet - File path in storage service - */ - fileReadStream(_bucketName = undefined, fileToGet = undefined) { - return async (req, res, next) => { - let bucketName = _bucketName; - let fileToGet = req.params.slug.replace("__", "/") + "/" + req.params.filename; - logger.info({ msg: "AWS__StorageService - fileReadStream called for bucketName " + bucketName + " for file " + fileToGet }); - - if (fileToGet.includes(".json")) { - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - await this.client - .send(this.getAWSCommand(bucketName, fileToGet, undefined)) - .then((resp) => { - streamToString(_.get(resp, "Body")) - .then((data) => { - res.end(data); - }) - .catch((err) => { - storageLogger.s500(res, "AWS__StorageService : readStream error - Error 500", err, "Failed to execute readStream"); - }); - }) - .catch((error) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - storageLogger.s404(res, "AWS__StorageService : readStream client send error - Error with status code 404", error, "File not found"); - } else { - storageLogger.s500(res, "AWS__StorageService : readStream client send error - Error 500", error, "Failed to display blob"); - } - }); - } else { - this.fileExists(bucketName, fileToGet, undefined, async (error, resp) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - storageLogger.s404(res, "AWS__StorageService : fileExists error - Error with status code 404", error, "File does not exists"); - } else if (_.get(resp, "$metadata.httpStatusCode") == 200) { - const command = this.getAWSCommand(bucketName, fileToGet, undefined); - // `expiresIn` - The number of seconds before the presigned URL expires - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: 3600 }); - const response = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: { - signedUrl: presignedURL, - }, - }; - res.status(200).send(this.apiResponse(response)); - } else { - storageLogger.s500(res, "AWS__StorageService : fileExists client send error - Error 500", "", "Failed to check file exists"); - } - }); - } - }; - } - - getFileProperties(_bucketName = undefined) { - return (req, res, next) => { - const bucketName = _bucketName; - const fileToGet = JSON.parse(req.query.fileNames); - logger.info({ msg: "AWS__StorageService - getFileProperties called for bucketName " + bucketName + " for file " + fileToGet }); - const responseData = {}; - if (Object.keys(fileToGet).length > 0) { - const getBlogRequest = []; - for (const [key, file] of Object.entries(fileToGet)) { - const req = { - bucketName: bucketName, - file: file, - reportname: key, - }; - getBlogRequest.push( - async.reflect((callback) => { - this.getBlobProperties(req, callback); - }) - ); - } - async.parallel(getBlogRequest, (err, results) => { - if (results) { - results.map((blob) => { - if (blob.error) { - responseData[_.get(blob, "error.reportname")] = blob.error; - } else { - responseData[_.get(blob, "value.reportname")] = { - lastModified: _.get(blob, "value.lastModified"), - reportname: _.get(blob, "value.reportname"), - statusCode: _.get(blob, "value.statusCode"), - fileSize: _.get(blob, "value.contentLength"), - }; - } - }); - const finalResponse = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: responseData, - }; - res.status(200).send(this.apiResponse(finalResponse)); - } - }); - } - }; - } - - async getBlobProperties(request, callback) { - this.fileExists(request.bucketName, request.file, undefined, (error, resp) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - logger.error({ msg: "AWS__StorageService : getBlobProperties_fileExists error - Error with status code 404. File does not exists - " + request.file, error: error }); - callback({ msg: _.get(error, "name"), statusCode: _.get(error, "$metadata.httpStatusCode"), filename: request.file, reportname: request.reportname }); - } else if (_.get(resp, "$metadata.httpStatusCode") == 200) { - resp.reportname = request.reportname; - resp.statusCode = 200; - logger.info({ - msg: "AWS__StorageService : getBlobProperties_fileExists success with status code 200. File does exists - " + request.file, - statusCode: _.get(error, "$metadata.httpStatusCode"), - }); - callback(null, resp); - } else { - logger.error({ msg: "AWS__StorageService : getBlobProperties_fileExists client send error - Error 500 Failed to check file exists" }); - callback(true); - } - }); - } - - async getFileAsText(container = undefined, fileToGet = undefined, callback) { - const bucketName = container; - logger.info({ msg: "AWS__StorageService : getFileAsText called for bucket " + bucketName + " for file " + fileToGet }); - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - await this.client - .send(this.getAWSCommand(bucketName, fileToGet)) - .then((resp) => { - streamToString(_.get(resp, "Body")) - .then((data) => { - callback(null, data); - }) - .catch((err) => { - logger.error({ msg: "AWS__StorageService : getFileAsText error - Error 500", err: "Failed to execute getFileAsText" }); - callback(err); - }); - }) - .catch((error) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - logger.error({ msg: "AWS__StorageService : getFileAsText client send error - Error with status code 404. File not found", error: error }); - } else { - logger.error({ msg: "AWS__StorageService : getFileAsText client send error - Error 500. Failed to display blob", error: error }); - } - callback(error); - }); - } - - blockStreamUpload(uploadContainer = undefined) { - return (req, res) => { - try { - const bucketName = uploadContainer; - const blobFolderName = new Date().toLocaleDateString(); - let form = new multiparty.Form(); - form.on("part", async (part) => { - if (part.filename) { - let size = part.byteCount - part.byteOffset; - let name = `${_.get(req, "query.deviceId")}_${Date.now()}.${_.get(part, "filename")}`; - logger.info({ - msg: "AWS__StorageService : blockStreamUpload Uploading file to bucket " + uploadContainer + " to folder " + blobFolderName + " for file name " + name + " with size " + size, - }); - let keyPath = uploadContainer + "/" + blobFolderName + "/" + name; - logger.info({ - msg: "AWS__StorageService : blockStreamUpload Uploading file to " + keyPath, - }); - try { - const parallelUploads3 = new Upload({ - client: this.client, - params: { Bucket: bucketName, Key: keyPath, Body: part }, - leavePartsOnError: false, - }); - parallelUploads3.on("httpUploadProgress", (progress) => { - let toStr; - for (let key in progress) { - if (progress.hasOwnProperty(key)) { - toStr += `${key}: ${progress[key]}` + ", "; - } - } - logger.info({ - msg: "AWS__StorageService : blockStreamUpload Uploading progress " + toStr, - }); - }); - await parallelUploads3 - .done() - .then((data) => { - const response = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: { - message: "Successfully uploaded to blob", - }, - }; - return res.status(200).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - }) - .catch((err) => { - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: "Failed to upload to blob", - }, - result: {}, - }; - logger.error({ - msg: "AWS__StorageService : blockStreamUpload parallelUploads3 Failed to upload desktop crash logs to blob", - error: err, - }); - return res.status(500).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - }); - } catch (e) { - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: "Failed to upload to blob", - }, - result: {}, - }; - logger.error({ - msg: "AWS__StorageService : blockStreamUpload try catch Failed to upload desktop crash logs to blob", - error: e, - }); - return res.status(500).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - } - } - }); - form.parse(req); - } catch (error) { - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: "Failed to upload to blob", - }, - result: {}, - }; - logger.error({ - msg: "AWS__StorageService : blockStreamUpload Failed to upload desktop crash logs to blob", - error: error, - }); - return res.status(500).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - } - }; - } - - apiResponse({ responseCode, result, params: { err, errmsg, status } }, id = "api.report") { - return { - id: id, - ver: "1.0", - ts: dateFormat(new Date(), "yyyy-mm-dd HH:MM:ss:lo"), - params: { - resmsgid: uuidv1(), - msgid: null, - status: status, - err: err, - errmsg: errmsg, - }, - responseCode: responseCode, - result: result, - }; - } - - async getPreSignedUrl(container, fileNames) { - const presignedURLs = await Promise.all( - fileNames.map(async (fileName) => { - const command = this.getAWSPutCommand(container, fileName, undefined); - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: 3600 }) - return { fileName, presignedURL }; - }) - ) - return presignedURLs - } - - /** - * @description - Function to get file names S3 bucket for a specific date range - * @param {string} container - Bucket name to fetch the files from - * @param {string} container_prefix - Prefix of the path if the files are nested - * @param {string} type - Folder name/Type of data to fetch the files for - * @param {string} dateRange - Range of time interval, to get the files for - * @param {string} datasetId - Dataset Id to fetch the files for - */ - async filterDataByRange(container, container_prefix, type, dateRange, datasetId) { - let startDate = moment(dateRange.from); - let endDate = moment(dateRange.to); - let result = []; - let promises = []; - for (let analysisDate = startDate; analysisDate <= endDate; analysisDate = analysisDate.add(1, "days")) { - promises.push(new Promise((resolve, reject) => { - const pathPrefix = `${container_prefix}/${type}/${datasetId}/${analysisDate.format("YYYY-MM-DD")}`; - try { - resolve(this.client.send(this.listAWSCommand(container, pathPrefix,))); - } - catch (err) { console.log(err) } - })) - } - let S3Objects = await Promise.all(promises); - S3Objects.map((S3Object) => { - S3Object.Contents?.map((content) => { - result.push((content.Key || "")); - }) - }); - return (result); - } - - /** - * @description - Function to get file download URLs from S3 bucket - * @param {string} container - Bucket name to fetch the files from - * @param {Array} filesList - List of file keys obtained for generating signed urls for download - */ - async getFilesSignedUrls(container, filesList) { - const signedUrlsPromises = filesList.map((fileNameWithPrefix) => { - return new Promise(async (resolve, reject) => { - const command = this.getAWSCommand(container, fileNameWithPrefix); - const fileName = fileNameWithPrefix.split("/").pop(); - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: globalConfig.exhaust_config.storage_url_expiry, }); - resolve({ [fileName]: presignedURL }); - }); - }); - const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles = {}; - const files = []; - // Formatting response - signedUrlsList.map(async (fileObject) => { - const fileDetails = _.keys(fileObject); - const fileUrl = _.values(fileObject)[0]; - const period = getFileKey(fileDetails[0]); - if(_.has(periodWiseFiles, period)) - periodWiseFiles[period].push(fileUrl); - else { - periodWiseFiles[period] = []; - periodWiseFiles[period].push(fileUrl); - } - files.push(fileUrl); - }); - return { - expiresAt: moment().add(globalConfig.exhaust_config.storage_url_expiry, 'seconds').toISOString(), - files, - periodWiseFiles, - }; - } - - /** - * @description - Function to get file names S3 bucket for a specific date range - * @param {string} container - Bucket name to fetch the files from - * @param {string} container_prefix - Prefix of the path if the files are nested - * @param {string} type - Folder name/Type of data to fetch the files for - * @param {string} dateRange - Range of time interval, to get the files for - * @param {string} datasetId - Dataset Id to fetch the files for - */ - async getFiles(container, container_prefix, type, dateRange, datasetId) { - const filesList = await this.filterDataByRange(container, container_prefix, type, dateRange, datasetId); - const signedUrlsList = await this.getFilesSignedUrls(container, filesList); - return signedUrlsList; - } -} - -module.exports = AWSStorageService; diff --git a/api-service/src/v1/lib/client-cloud-services/AzureStorageService.js b/api-service/src/v1/lib/client-cloud-services/AzureStorageService.js deleted file mode 100644 index 4c2e053c..00000000 --- a/api-service/src/v1/lib/client-cloud-services/AzureStorageService.js +++ /dev/null @@ -1,405 +0,0 @@ -/** - * @file - Azure Storage Service - * @exports - `AzureStorageService` - * @since - 5.0.1 - * @version - 2.0.0 - * @implements - BaseStorageService - * - * @see {@link https://learn.microsoft.com/en-us/javascript/api/@azure/storage-blob/?view=azure-node-latest | Azure Blob Documentation} - * @see {@link https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/storage-blob/MigrationGuide.md#uploading-a-blob-to-the-container | Azure Migration Guide} - */ - -const BaseStorageService = require("./BaseStorageService"); -const { logger } = require("@project-sunbird/logger"); -const _ = require("lodash"); -const { TextDecoder } = require("util"); -const { config: globalConfig } = require("../../configs/Config"); -const moment = require("moment"); -const { - BlobServiceClient, - StorageSharedKeyCredential, - generateBlobSASQueryParameters, - ContainerClient, -} = require("@azure/storage-blob"); -const { getFileKey } = require("../../utils/common"); -const READ = "r"; - -class AzureStorageService extends BaseStorageService { - constructor(config) { - super(); - if (!_.get(config, "identity") || !_.get(config, "credential")) { - throw new Error( - "Azure__StorageService :: Required configuration is missing" - ); - } - try { - this.sharedKeyCredential = new StorageSharedKeyCredential( - config?.identity, - config?.credential - ); - this.blobService = new BlobServiceClient( - `https://${config?.identity}.blob.core.windows.net`, - this.sharedKeyCredential - ); - this.containerClient = new ContainerClient( - `https://${config?.identity}.blob.core.windows.net/${globalConfig?.exhaust_config?.container}`, - this.sharedKeyCredential - ); - } catch (error) { - logger.info({ - msg: "Azure__StorageService - Unable to create Azure client", - }); - } - } - - async fileExists(container, fileToGet, callback) { - if (!container || !fileToGet || !callback) - throw new Error("Invalid arguments"); - logger.info({ - msg: - "Azure__StorageService - fileExists called for container " + - container + - " for file " + - fileToGet, - }); - const blobClient = this.blobService - .getContainerClient(container) - .getBlobClient(fileToGet); - try { - const blobProperties = await blobClient.getProperties(); - if (blobProperties) { - const response = { - exists: true, - }; - callback(null, response); - } - } catch (error) { - callback(error); - } - } - - /** - * @description - Retrieves a shared access signature token - * @param { string } container - Container name - * @param { string } blob - Blob to be fetched - * @param { azure.common.SharedAccessPolicy } sharedAccessPolicy - Shared access policy - * @param { azure.common.ContentSettingsHeaders } headers - Optional header values to set for a blob returned wth this SAS - * @return { string } - The shared access signature - */ - generateSharedAccessSignature( - container, - blob, - sharedAccessPolicy, - headers - ) { - const sasToken = generateBlobSASQueryParameters( - { - containerName: container, - blobName: blob, - ...sharedAccessPolicy.AccessPolicy, - ...headers, - }, - this.sharedKeyCredential - ).toString(); - return sasToken; - } - - /** - * @description - Retrieves a blob or container URL - * @param { string } container - Container name - * @param { string } blob - Blob to be fetched - * @param { string } SASToken - Shared Access Signature token - * @return { string } - Formatted URL string - */ - getUrl(container, blob, SASToken) { - const blobClient = this.blobService - .getContainerClient(container) - .getBlobClient(blob); - return `${blobClient.url}?${SASToken}`; - } - - async getBlobProperties(request, callback) { - logger.info({ - msg: - "Azure__StorageService - getBlobProperties called for container " + - request.container + - " for file " + - request.file, - }); - const blobClient = this.blobService - .getContainerClient(request.container) - .getBlobClient(request.file); - try { - const blobProperties = await blobClient.getProperties(); - if (blobProperties) { - blobProperties.reportname = request.reportname; - blobProperties.filename = request.file; - blobProperties.statusCode = 200; - callback(null, blobProperties); - } - } catch (error) { - logger.error({ - msg: "Azure__StorageService : readStream error - Error with status code 404", - }); - callback({ - msg: "NotFound", - statusCode: error.statusCode, - filename: request.file, - reportname: request.reportname, - }); - } - } - - async getFileAsText( - container = undefined, - fileToGet = undefined, - prefix = undefined, - callback - ) { - const blobClient = this.blobService - .getContainerClient(container) - .getBlobClient(fileToGet); - try { - const downloadResponse = await blobClient.download(0); - const textDecoder = new TextDecoder("utf-8"); - const content = []; - for await (const chunk of downloadResponse.readableStreamBody) { - content.push(textDecoder.decode(chunk)); - } - const text = content.join(""); - logger.info({ - msg: - "Azure__StorageService : getFileAsText success for container " + - container + - " for file " + - fileToGet, - }); - callback(null, text); - } catch (error) { - logger.error({ - msg: "Azure__StorageService : getFileAsText error => ", - error, - }); - delete error.request; - delete error.response; - delete error.details; - callback(error); - } - } - - upload(container, fileName, filePath, callback) { - throw new Error("AzureStorageService :: upload() must be implemented"); - } - - async getPreSignedUrl(container, fileName, prefix = undefined) { - if (prefix) { - fileName = prefix + fileName; - } - const presignedURL = await this.getSignedUrl( - container, - fileName, - globalConfig.exhaust_config.storage_url_expiry - ); - return presignedURL; - } - - /** - * @description - Generates a pre-signed URL for a specific operation on a file in Azure storage. - * @param {string} container - Azure container or bucket name. - * @param {string} filePath - Path to the file within the container. - * @param {number} expiresIn - Optional. Number of seconds before the pre-signed URL expires. - * @param {string} permission - Optional. The permission for the operation (e.g., READ, WRITE). - * @returns {Promise} - A promise that resolves to the pre-signed URL. - */ - getSignedUrl(container, filePath, expiresIn = 3600, permission = "") { - let startDate = new Date(); - let expiryDate = new Date(startDate); - expiryDate.setMinutes(startDate.getMinutes() + expiresIn); - startDate.setMinutes(startDate.getMinutes() - expiresIn); - let sharedAccessPolicy = { - AccessPolicy: { - permissions: permission !== "" ? permission : READ, - startsOn: startDate, - expiresOn: expiryDate, - }, - }; - let azureHeaders = {}; - let token = this.generateSharedAccessSignature( - container, - filePath, - sharedAccessPolicy, - azureHeaders - ); - let sasUrl = this.getUrl(container, filePath, token); - return Promise.resolve(sasUrl); - } - - /** - * @description - Generates a pre-signed URL for downloading a file from the Azure storage. - * @param {string} container - Azure container or bucket name. - * @param {string} filePath - Path to the file within the container. - * @param {number} expiresIn - Optional. Number of seconds before the pre-signed URL expires. - * @returns {Promise} - A promise that resolves to the downloadable URL. - */ - getDownloadableUrl(container, filePath, expiresIn = 3600) { - let startDate = new Date(); - let expiryDate = new Date(startDate); - expiryDate.setMinutes(startDate.getMinutes() + expiresIn); - let sharedAccessPolicy = { - AccessPolicy: { - permissions: READ, - startsOn: startDate, - expiresOn: expiryDate, - }, - }; - let azureHeaders = {}; - let token = this.generateSharedAccessSignature( - container, - filePath, - sharedAccessPolicy, - azureHeaders - ); - let downloadableUrl = this.getUrl(container, filePath, token); - return Promise.resolve(downloadableUrl); - } - - /** - * @description - Generates a ingestion specification for a file. - * @param {string} container - Bucket name. - * @param {string} filePath - Path to the file in the bucket. - * @returns {Promise} - A Promise that resolves to the Druid ingestion specification. - */ - getFileUrlForIngestion(container, filePath) { - let druidSpec = { - type: "azure", - uris: [`azure://${container}/${filePath}`], - }; - return Promise.resolve(druidSpec); - } - - /** - * @description - Function to get file download URLs from S3 bucket - * @param {string} container - Bucket name to fetch the files from - * @param {Array} filesList - List of file keys obtained for generating signed urls for download - */ - async getFilesSignedUrls(container, filesList) { - const signedUrlsPromises = filesList.map((fileNameWithPrefix) => { - return new Promise(async (resolve, reject) => { - const presignedURL = await this.getPreSignedUrl( - container, - fileNameWithPrefix - ); - const fileName = fileNameWithPrefix.split("/").pop(); - resolve({ [fileName]: presignedURL }); - }); - }); - const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles = {}; - const files = []; - // Formatting response - signedUrlsList.map(async (fileObject) => { - const fileDetails = _.keys(fileObject); - const fileUrl = _.values(fileObject)[0]; - const period = getFileKey(fileDetails[0]); - if (_.has(periodWiseFiles, period)) - periodWiseFiles[period].push(fileUrl); - else { - periodWiseFiles[period] = []; - periodWiseFiles[period].push(fileUrl); - } - files.push(fileUrl); - }); - return { - expiresAt: moment() - .add(globalConfig.exhaust_config.storage_url_expiry, "seconds") - .toISOString(), - files, - periodWiseFiles, - }; - } - - /** - * @description - Function to get file names from container for a specific date range - * @param {string} container - container name to fetch the files from - * @param {string} container_prefix - Prefix of the path if the files are nested - * @param {string} type - Folder name/Type of data to fetch the files for - * @param {string} dateRange - Range of time interval, to get the files for - * @param {string} datasetId - Dataset Id to fetch the files for - */ - async filterDataByRange( - container, - container_prefix, - type, - dateRange, - datasetId - ) { - let startDate = moment(dateRange.from); - let endDate = moment(dateRange.to); - let result = []; - let promises = []; - for ( - let analysisDate = startDate; - analysisDate <= endDate; - analysisDate = analysisDate.add(1, "days") - ) { - promises.push( - new Promise(async (resolve, reject) => { - const pathPrefix = `${container_prefix}/${type}/${datasetId}/${analysisDate.format( - "YYYY-MM-DD" - )}`; - try { - const items = this.containerClient.listBlobsByHierarchy( - "/", - { prefix: pathPrefix } - ); - for await (const item of items) { - if (item && item.kind === "blob") resolve(item); - else resolve(null); - return; - } - } catch (err) { - console.log( - `Unable to list the blobs present in directory ${pathPrefix}` - ); - console.log(err); - reject(err); - return; - } - }) - ); - } - try { - result = await Promise.all(promises); - if(result.length > 0) result = result.map((item) => item.name); - return result; - } catch (err) { - console.log(err); - return []; - } - } - - /** - * @description - Function to get file names S3 bucket for a specific date range - * @param {String} container - Bucket name to fetch the files from - * @param {String} container_prefix - Prefix of the path if the files are nested - * @param {String} type - Folder name/Type of data to fetch the files for - * @param {String} dateRange - Range of time interval, to get the files for - * @param {String} datasetId - Dataset Id to fetch the files for - */ - async getFiles(container, container_prefix, type, dateRange, datasetId) { - const filesList = await this.filterDataByRange( - container, - container_prefix, - type, - dateRange, - datasetId - ); - const signedUrlsList = await this.getFilesSignedUrls( - container, - filesList - ); - return signedUrlsList; - } -} - -module.exports = AzureStorageService; diff --git a/api-service/src/v1/lib/client-cloud-services/BaseStorageService.js b/api-service/src/v1/lib/client-cloud-services/BaseStorageService.js deleted file mode 100644 index 0db90e37..00000000 --- a/api-service/src/v1/lib/client-cloud-services/BaseStorageService.js +++ /dev/null @@ -1,109 +0,0 @@ -/** - * @file - Base Cloud Storage Service - * @description - Provides the interface and base implementation of StorageService - * @module - BaseStorageService - * @exports - `BaseStorageService` - * @since - 5.0.1 - * @version - 1.0.0 - */ - -class BaseStorageService { - - /** - * @description - Download file from storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {String} container - Container name or folder name in storage service - * @param {String} fileToDownload - File path in storage service - * @param {Boolean} isDirectory - default `false` - */ - downloadFile(container, fileToDownload, isDirectory = false) { - throw new Error('BaseStorageService :: downloadFile() must be implemented'); - } - - /** - * @description - Retrieves a blob or container URL - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - getURI(container, filePath) { - throw new Error('BaseStorageService :: getURI() must be implemented'); - } - - /** - * @description - Provides a stream to read from a storage - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - fileReadStream(container, filePath) { - throw new Error('BaseStorageService :: fileReadStream() must be implemented'); - } - - /** - * @description - Checks whether or not a blob exists on the storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - fileExists(container, filePath) { - throw new Error('BaseStorageService :: fileExists() must be implemented'); - } - - /** - * @description - Retrieves a shared access signature token or signed URL - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - getSharedAccessSignature(container, filePath) { - throw new Error('BaseStorageService :: getSharedAccessSignature() must be implemented'); - } - - /** - * @description - Get all user-defined metadata, standard HTTP properties, and system properties for the blob - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - getFileProperties(container, filePath) { - throw new Error('BaseStorageService :: getFileProperties() must be implemented'); - } - - /** - * @description - Downloads a blob / file into a text string - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - * @param { function } callback - Callback function - */ - getFileAsText(container, filePath, callback) { - throw new Error('BaseStorageService :: getFileAsText() must be implemented'); - } - - /** - * @description - Stream upload file to storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - * @param { function } callback - Callback function - */ - blockStreamUpload(container, filePath, callback) { - throw new Error('BaseStorageService :: blockStreamUpload() must be implemented'); - } - - /** - * @description - Upload file to storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} fileName - File name of file to be uploaded - * @param {string} filePath - File path for file to be uploaded - * @param {object} options - Metadata options for file - * @param { function } callback - Callback function - */ - upload(container, fileName, filePath, callback) { - throw new Error('BaseStorageService :: upload() must be implemented'); - } -} - -module.exports = BaseStorageService; \ No newline at end of file diff --git a/api-service/src/v1/lib/client-cloud-services/GCPStorageService.js b/api-service/src/v1/lib/client-cloud-services/GCPStorageService.js deleted file mode 100644 index 4efb2245..00000000 --- a/api-service/src/v1/lib/client-cloud-services/GCPStorageService.js +++ /dev/null @@ -1,278 +0,0 @@ -/** - * @file - Google Cloud Provider (GCP) Storage Service - * @exports - `GCPStorageService` - * @since - 5.0.1 - * @version - 1.0.0 - * @implements - BaseStorageService - * @see {@link https://googleapis.dev/nodejs/storage/latest/Bucket.html | GCloud Bucket} - */ - -const BaseStorageService = require("./BaseStorageService"); -const storageLogger = require("./storageLogger"); -const { Storage } = require("@google-cloud/storage"); -const { logger } = require("@project-sunbird/logger"); -const async = require("async"); -const _ = require("lodash"); -const dateFormat = require("dateformat"); -const uuidv1 = require("uuid/v1"); - -class GCPStorageService extends BaseStorageService { - constructor(config) { - super(); - if (!_.get(config, "identity")) { - throw new Error("GCLOUD__StorageService :: Required configuration is missing - [identity]"); - } - if (!_.get(config, "credential")) { - throw new Error("GCLOUD__StorageService :: Required configuration is missing - [credential]"); - } - if (!_.get(config, "projectId")) { - throw new Error("GCLOUD__StorageService :: Required configuration is missing - [projectId]"); - } - this._storage = new Storage({ - credentials: { - client_email: _.get(config, "identity"), - private_key: _.get(config, "credential")?.toString(), - }, - projectId: _.get(config, "projectId"), - }); - } - - fileExists(bucketName, fileToGet, prefix = "", cb) { - const file = this._storage.bucket(bucketName).file(prefix + fileToGet); - logger.info({ msg: "GCLOUD__StorageService - fileExists called for bucketName " + bucketName + " for file " + prefix + fileToGet }); - file.exists((err, exists) => { - if (err) cb(err); - if (exists) { - cb(null, exists); - } else { - cb(null, null); - } - }); - } - - /** - * @description - Provides a stream to read from a storage - * @param {string} bucketName - Bucket name or folder name in storage service - * @param {string} fileToGet - File path in storage service - */ - fileReadStream(_bucketName = undefined, fileToGet = undefined) { - return async (req, res, next) => { - let bucketName = _bucketName; - let fileToGet = _bucketName + req.params.slug.replace("__", "/") + "/" + req.params.filename; - logger.info({ msg: "GCLOUD__StorageService - fileReadStream called for bucketName " + bucketName + " for file " + fileToGet }); - - if (fileToGet.includes(".json")) { - try { - const file = this._storage.bucket(bucketName).file(fileToGet); - const fileStream = file.createReadStream(); - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - streamToString(fileStream) - .then((data) => { - res.end(data); - }) - .catch((err) => { - if (_.get(err, "code") === 404) { - storageLogger.s404(res, "GCLOUD__StorageService : readStream error - Error " + _.get(err, "code") + " " + _.get(err, "message"), "", _.get(err, "message")); - } else { - storageLogger.s500(res, "GCLOUD__StorageService : readStream client send error - Error 500", err, "Failed to display blob"); - } - }); - } catch (error) { - storageLogger.s500(res, "GCLOUD__StorageService : readStream client send error - Error 500", error, "Failed to display blob"); - } - } else { - this.fileExists(bucketName, fileToGet, "", (error, fileExists) => { - if (error) { - storageLogger.s404(res, "GCLOUD__StorageService : readStream_fileExists error - Error 404", error, "File does not exists"); - } else if (fileExists) { - this.getSharedAccessSignature(bucketName, fileToGet, "", undefined, (err, presignedURL) => { - if (err) { - storageLogger.s500(res, "GCLOUD__StorageService : readStream_getSharedAccessSignature - Error 500. Failed to get shared access signature", err, err); - } else { - const response = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: { - signedUrl: presignedURL, - }, - }; - logger.info({ msg: "GCLOUD__StorageService - readStream_getSharedAccessSignature called for bucketName " + bucketName + " for file " + fileToGet }); - res.status(200).send(this.apiResponse(response)); - } - }); - } else { - storageLogger.s500(res, "GCLOUD__StorageService : readStream_fileExists error - Error 500. Failed to fetch or File does not exists", error, "Failed to fetch or File does not exists"); - } - }); - } - }; - } - - async getSharedAccessSignature(bucketName, fileToGet, prefix = "", expiresIn, cb) { - let expiryDate; - if (!expiresIn) { - let startDate = new Date(); - expiryDate = new Date(startDate); - expiryDate.setMinutes(startDate.getMinutes() + 3600); - startDate.setMinutes(startDate.getMinutes() - 3600); - } else { - expiryDate = expiresIn; - } - const _config = { action: "read", expires: expiryDate }; - const file = this._storage.bucket(bucketName).file(prefix + fileToGet); - await file - .getSignedUrl(_config) - .then((signedUrl) => { - cb(null, signedUrl && signedUrl.length > 0 && signedUrl[0]); - }) - .catch((err) => cb(_.get(err, "message"))); - } - - getFileProperties(_bucketName = undefined) { - return (req, res, next) => { - const bucketName = _bucketName; - const fileToGet = JSON.parse(req.query.fileNames); - logger.info({ msg: "GCLOUD__StorageService - getFileProperties called for bucketName " + bucketName + " for file " + fileToGet }); - const responseData = {}; - if (Object.keys(fileToGet).length > 0) { - const getBlogRequest = []; - for (const [key, file] of Object.entries(fileToGet)) { - const req = { - bucketName: bucketName, - file: file, - reportname: key, - }; - getBlogRequest.push( - async.reflect((callback) => { - this.getBlobProperties(req, callback); - }) - ); - } - async.parallel(getBlogRequest, (err, results) => { - if (results) { - results.map((blob) => { - if (blob.error) { - responseData[_.get(blob, "error.reportname")] = blob.error; - } else { - responseData[_.get(blob, "value.reportname")] = { - lastModified: _.get(blob, "value.updated"), - reportname: _.get(blob, "value.reportname"), - statusCode: _.get(blob, "value.statusCode"), - fileSize: _.get(blob, "value.size"), - }; - } - }); - const finalResponse = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: responseData, - }; - res.status(200).send(this.apiResponse(finalResponse)); - } - }); - } - }; - } - - async getBlobProperties(request, callback) { - const file = this._storage.bucket(request.bucketName).file(request.file); - file.getMetadata((err, metadata, resp) => { - if (err) { - logger.error({ msg: "GCLOUD__StorageService : getBlobProperties_getMetadata client send error - Error 500 Failed to check file exists", err: err }); - callback(err); - } else if (_.get(resp, "statusCode") == 404) { - logger.error({ msg: "GCLOUD__StorageService : getBlobProperties_getMetadata error - Error with status code 404. File does not exists - " + request.file, error: resp }); - callback({ msg: _.get(resp, "statusMessage"), statusCode: _.get(resp, "statusCode"), filename: request.file, reportname: request.reportname }); - } else if (_.get(resp, "statusCode") == 200) { - metadata.reportname = request.reportname; - metadata.statusCode = 200; - logger.info({ - msg: "GCLOUD__StorageService : getBlobProperties_getMetadata success with status code 200. File exists - " + request.file, - statusCode: _.get(resp, "statusCode"), - }); - callback(null, metadata); - } else { - logger.error({ msg: "GCLOUD__StorageService : getBlobProperties_getMetadata client send error - Error 500 Failed to check file exists" }); - callback(true); - } - }); - } - - async getFileAsText(container = undefined, fileToGet = undefined, callback) { - const bucketName = container; - logger.info({ msg: "GCLOUD__StorageService : getFileAsText called for bucket " + bucketName + " for file " + fileToGet }); - const file = this._storage.bucket(bucketName).file(container + fileToGet); - logger.info({ msg: "GCLOUD__StorageService : getFileAsText called for bucket " + bucketName + " for file " + container + fileToGet }); - const fileStream = file.createReadStream(); - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - streamToString(fileStream) - .then((data) => { - callback(null, data); - }) - .catch((err) => { - if (_.get(err, "code") === 404) { - callback(err); - logger.error({ msg: "GCLOUD__StorageService : getFileAsText error - Error " + _.get(err, "code") + " " + _.get(err, "message") }); - } else { - callback({ err: "Failed to display blob", statusCode: 500 }); - logger.error({ msg: "GCLOUD__StorageService : getFileAsText client send error - Error 500. Failed to display blob, Error ", err }); - } - }); - } - - blockStreamUpload(uploadContainer = undefined) { - return (req, res) => { - logger.info({ msg: "GCLOUD__StorageService : blockStreamUpload called for bucket" }); - return res.status(200); - }; - } - - apiResponse({ responseCode, result, params: { err, errmsg, status } }) { - return { - id: "api.report", - ver: "1.0", - ts: dateFormat(new Date(), "yyyy-mm-dd HH:MM:ss:lo"), - params: { - resmsgid: uuidv1(), - msgid: null, - status: status, - err: err, - errmsg: errmsg, - }, - responseCode: responseCode, - result: result, - }; - } - upload(container, fileName, filePath, callback) { - throw new Error("BaseStorageService :: upload() must be implemented"); - } -} -module.exports = GCPStorageService; diff --git a/api-service/src/v1/lib/client-cloud-services/index.js b/api-service/src/v1/lib/client-cloud-services/index.js deleted file mode 100644 index abf31642..00000000 --- a/api-service/src/v1/lib/client-cloud-services/index.js +++ /dev/null @@ -1,34 +0,0 @@ -/** - * @file - Entry file referencing Storage Service - * @description - Entry file referencing Storage Service - * @exports - `AzureStorageService`, `AWSStorageService` and 'GCPStorageService` - * @author - RAJESH KUMARAVEL - * @since - 5.0.3 - * @version - 1.0.0 - */ - -const AzureStorageService = require("./AzureStorageService"); -const AWSStorageService = require("./AWSStorageService"); -const GCPStorageService = require("./GCPStorageService"); - -/** - * Based on Environment Cloud Provider value - * Export respective Storage Service - */ - -function init(provider) { - switch (provider) { - case "azure": - return AzureStorageService; - break; - case "aws": - return AWSStorageService; - break; - case "gcloud": - return GCPStorageService; - break; - default: - throw new Error(`Client Cloud Service - ${provider} provider is not supported`); - } -} -module.exports = { init }; diff --git a/api-service/src/v1/lib/client-cloud-services/storageLogger.js b/api-service/src/v1/lib/client-cloud-services/storageLogger.js deleted file mode 100644 index 506036db..00000000 --- a/api-service/src/v1/lib/client-cloud-services/storageLogger.js +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @file - Storage Service logger - * @exports - Different HTTP status log builder(s) - * @since - 5.0.1 - * @version - 1.0.0 - */ - -const { logger } = require('@project-sunbird/logger'); -const dateFormat = require('dateformat'); -const uuidv1 = require('uuid/v1'); - -module.exports = { - 's500': (res, logMessage = '', stack = '', errorMessage) => { - logger.error({ msg: logMessage, error: stack }); - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: errorMessage - }, - result: {} - } - res.status(500).send(apiResponse(response)); - }, - 's404': (res, logMessage = '', stack = '', errorMessage) => { - logger.error({ msg: logMessage, error: stack }); - const response = { - responseCode: "CLIENT_ERROR", - params: { - err: "CLIENT_ERROR", - status: "failed", - errmsg: errorMessage - }, - result: {} - } - res.status(404).send(apiResponse(response)); - } -}; - -function apiResponse({ responseCode, result, params: { err, errmsg, status } }) { - return { - 'id': 'api.report', - 'ver': '1.0', - 'ts': dateFormat(new Date(), 'yyyy-mm-dd HH:MM:ss:lo'), - 'params': { - 'resmsgid': uuidv1(), - 'msgid': null, - 'status': status, - 'err': err, - 'errmsg': errmsg - }, - 'responseCode': responseCode, - 'result': result - } -}; diff --git a/api-service/src/v1/lib/dispatcher/dispatcher.js b/api-service/src/v1/lib/dispatcher/dispatcher.js deleted file mode 100644 index a2f2eb4d..00000000 --- a/api-service/src/v1/lib/dispatcher/dispatcher.js +++ /dev/null @@ -1,54 +0,0 @@ -const winston = require("winston"); -const { KafkaDispatcher } = require("./kafka-dispatcher"); -require("winston-daily-rotate-file"); -require("./kafka-dispatcher"); -const appConfig = require("../../configs/Config") -const defaultFileOptions = { - filename: "dispatcher-%DATE%.log", - datePattern: "YYYY-MM-DD", - maxsize: appConfig.config.telemetry_service_config.maxsize, - maxFiles: "100", - zippedArchive: true, - json: true, -}; - - -class Dispatcher { - constructor(options) { - if (!options) throw new Error("Dispatcher options are required"); - this.logger = new winston.Logger({ level: "info" }); - this.options = options; - this.kafkaDispatcher = new KafkaDispatcher(this.options) - if (this.options.dispatcher == "kafka") { - this.logger.add(winston.transports.Kafka, this.options); - console.log("Kafka transport enabled !!!"); - } else if (this.options.dispatcher == "file") { - const config = Object.assign(defaultFileOptions, this.options); - this.logger.add(winston.transports.DailyRotateFile, config); - console.log("File transport enabled !!!"); - } else { - // Log to console - this.options.dispatcher = "console"; - const config = Object.assign({ json: true, stringify: (obj) => JSON.stringify(obj) }, this.options); - this.logger.add(winston.transports.Console, config); - console.log("Console transport enabled !!!"); - } - } - - dispatch(mid, message, params, callback) { - // this.logger.log("info", message, params, callback) - this.kafkaDispatcher.log("info", message, params, callback); - } - - health(callback) { - if (this.options.dispatcher === "kafka") { - this.logger.transports["kafka"].health(callback); - } else if (this.options.dispatcher === "console") { - callback(null, true); - } else { - callback("telemetry service is not healthy"); - } - } -} - -module.exports = { Dispatcher }; diff --git a/api-service/src/v1/lib/dispatcher/kafka-dispatcher.js b/api-service/src/v1/lib/dispatcher/kafka-dispatcher.js deleted file mode 100644 index 2da3e953..00000000 --- a/api-service/src/v1/lib/dispatcher/kafka-dispatcher.js +++ /dev/null @@ -1,60 +0,0 @@ -const winston = require("winston"); -const { Kafka } = require("kafkajs"); -const _ = require("lodash"); - -class KafkaDispatcher extends winston.Transport { - constructor(options) { - super(); - this.name = "kafka"; - this.options = options.kafka; - if (this.options.compression_type == "snappy") { - this.compression_attribute = 2; - } else if (this.options.compression_type == "gzip") { - this.compression_attribute = 1; - } else { - this.compression_attribute = 0; - } - this.client = new Kafka(this.options.config); - this.producer = this.client.producer(); - this.init(); - } - async init() { - await this.producer - .connect() - .then(() => { - console.log("kafka dispatcher is ready"); - }) - .catch((err) => { - console.error("Unable to connect to kafka", err.message); - }); - } - async log(level, msg, kafkaTopics, callback) { - try { - console.log(`pushing events to topic '${kafkaTopics[0]}'...`); - await this.producer.send({ - topic: kafkaTopics[0], - messages: [{ value: msg }], - }); - callback(null, true); - } catch (err) { - console.log(err); - callback(err); - } - } - - async health(callback) { - try { - const admin = this.client.admin(); - await admin.connect(); - await admin.listTopics(); - await admin.disconnect(); - callback(null, true); - } catch (error) { - callback(error); - } - } -} - -winston.transports.Kafka = KafkaDispatcher; - -module.exports = { KafkaDispatcher }; diff --git a/api-service/src/v1/lib/envVariables.js b/api-service/src/v1/lib/envVariables.js deleted file mode 100644 index b7bd5c6b..00000000 --- a/api-service/src/v1/lib/envVariables.js +++ /dev/null @@ -1,15 +0,0 @@ -const envVariables = { - level: process.env.telemetry_log_level || 'info', - localStorageEnabled: process.env.telemetry_local_storage_enabled || 'true', - dispatcher: process.env.telemetry_local_storage_type || 'kafka', - telemetryProxyEnabled: process.env.telemetry_proxy_enabled, - proxyURL: process.env.telemetry_proxy_url, - proxyAuthKey: process.env.telemetry_proxy_auth_key, - kafkaHost: `${process.env.kafka_host || 'localhost'}:${process.env.kafka_port || 9092}`, - topic: `${process.env.system_env || 'local'}.ingest`, - compression_type: process.env.telemetry_kafka_compression || 'none', - filename: process.env.telemetry_file_filename || 'telemetry-%DATE%.log', - maxsize: process.env.telemetry_file_maxsize || 10485760, - maxFiles: process.env.telemetry_file_maxfiles || '100', -} -module.exports = envVariables; \ No newline at end of file diff --git a/api-service/src/v1/lib/services/TelemetryService.js b/api-service/src/v1/lib/services/TelemetryService.js deleted file mode 100644 index a25679c6..00000000 --- a/api-service/src/v1/lib/services/TelemetryService.js +++ /dev/null @@ -1,124 +0,0 @@ -const uuidv1 = require("uuid/v1"), - // request = require("request"), - DispatcherClass = require("../dispatcher/dispatcher").Dispatcher; -const axios = require('axios').default; -const config = require("../../configs/Config").config; -const responseHandler = require("../../helpers/ResponseHandler").ResponseHandler; - -// TODO: Make this efficient. Implementation to be similar to typesafe config. Right now one configuration holds -// together all supported transport configurations - -class TelemetryService { - constructor(Dispatcher, config) { - this.config = config; - this.dispatcher = this.config.localStorageEnabled === "true" ? new Dispatcher(config) : undefined; - } - dispatch(req, res, ...params) { - const message = req.body; - message.did = req.get("x-device-id"); - message.channel = req.get("x-channel-id"); - message.pid = req.get("x-app-id"); - if (!message.mid) message.mid = uuidv1(); - message.syncts = new Date().getTime(); - - // add obsrv meta - const source = {meta: {id: "", connector_type: "api", version: config.version, entry_source: "api"}, trace_id: uuidv1()}; - const obsrvMeta = {syncts: new Date().getTime(), processingStartTime: new Date().getTime(), flags: {}, timespans: {}, error: {}, source: source}; - message.obsrv_meta = obsrvMeta; - - const data = JSON.stringify(message); - if (this.config.localStorageEnabled === "true" || this.config.telemetryProxyEnabled === "true") { - if (this.config.localStorageEnabled === "true" && this.config.telemetryProxyEnabled !== "true") { - // Store locally and respond back with proper status code - // this.dispatcher.dispatch(message.mid, data, this.getRequestCallBack(req, res)); - return new Promise(async (resolve, reject) => { - await this.dispatcher.dispatch(message.mid, data, params, (err, response) => { - if (err) { - return reject(err); - } else { - return resolve(response); - } - }); - }); - } else if (this.config.localStorageEnabled === "true" && this.config.telemetryProxyEnabled === "true") { - // Store locally and proxy to the specified URL. If the proxy fails ignore the error as the local storage is successful. Do a sync later - const options = this.getProxyRequestObj(req, data); - // request.post(options, (err, data) => { - // if (err) console.error("Proxy failed:", err); - // else console.log("Proxy successful! Server responded with:", data.body); - // }); - // this.dispatcher.dispatch(message.mid, data, this.getRequestCallBack(req, res)); - axios.post(options.url, options.body, { - headers: options.headers, - }).then((response) => { - console.log("Proxy successful! Server responded with:", data.body); - this.dispatcher.dispatch(message.mid, data, () => {}); - this.sendSuccess(req, res, { message: "the data has been successfully ingested" }); - }).catch((err) => { - console.error("Proxy failed:", err); - this.dispatcher.dispatch(message.mid, data, () => {}); - this.sendError(req, res, { message: err.message }); - }); - } else if (this.config.localStorageEnabled !== "true" && this.config.telemetryProxyEnabled === "true") { - // Just proxy - const options = this.getProxyRequestObj(req, data); - axios.post(options.url, options.body, { - headers: options.headers, - }).then( - this.sendSuccess(req, res, { message: "the data has been successfully ingested" }) - ).catch((err) => - this.sendError(req, res, { message: err.message }) - ) - // request.post(options, this.getRequestCallBack(req, res)); - } - } else { - this.sendError(req, res, { message: "Configuration error" }); - } - } - health(req, res) { - if (this.config.localStorageEnabled === "true") { - return new Promise(async (resolve, reject) => { - await this.dispatcher.health((err, response) => { - if (err) { - return reject(err); - } else { - return resolve(response); - } - }); - }); - } else if (this.config.telemetryProxyEnabled === "true") { - this.sendSuccess(req, res, { message: "Telemetry Proxy enabled" }); - } else { - this.sendError(req, res, { message: "Configuration error" }); - } - } - getRequestCallBack(req, res) { - let counter = 0; - return (err, data) => { - counter++; - if (counter > 1) return; - if (err) { - this.sendError(req, res, { message: err.message }); - } else { - this.sendSuccess(req, res, { message: "the data has been successfully ingested" }); - } - }; - } - sendError(req, res, options) { - responseHandler.errorResponse({ statusCode: 500, message: options.message, errCode: ["INTERNAL_SERVER_ERROR"] }, req, res); - } - sendSuccess(req, res, options) { - responseHandler.successResponse(req, res, { message: options.message }); - } - getProxyRequestObj(req, data) { - const headers = { authorization: "Bearer " + config.proxyAuthKey }; - if (req.get("content-type")) headers["content-type"] = req.get("content-type"); - if (req.get("content-encoding")) headers["content-encoding"] = req.get("content-encoding"); - return { - url: this.config.proxyURL, - headers: headers, - body: data, - }; - } -} -module.exports = new TelemetryService(DispatcherClass, config.telemetry_service_config); diff --git a/api-service/src/v1/managers/Extensions.ts b/api-service/src/v1/managers/Extensions.ts deleted file mode 100644 index 7e97f8c1..00000000 --- a/api-service/src/v1/managers/Extensions.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Application } from "express"; -import { extensions } from "../configs/Extensions"; -import path from "path"; - -export const loadExtensions = async (app: Application) => { - await loadRoutes(app) -} - -const loadRoutes = async (app: Application) => { - for (const extension of extensions) { - try { - const extensionPath = path.join(__dirname, "..", "..", "..", "node_modules", extension.id, extension.routePath) - let ExtensionRoute: any = await import(extensionPath) - const extensionRoute: IRouter = new ExtensionRoute.Router() - extensionRoute.init(app) - } catch (error) { - console.error(`Error while loading the extension of id ${extension.id} with path ${extension.routePath}`, error) - } - } -} - -export interface IRouter { - init(app: Application): void; - } - -export interface IRouterConstructor { - new(): IRouter; - } \ No newline at end of file diff --git a/api-service/src/v1/models/ConfigModels.ts b/api-service/src/v1/models/ConfigModels.ts deleted file mode 100644 index 538685aa..00000000 --- a/api-service/src/v1/models/ConfigModels.ts +++ /dev/null @@ -1,44 +0,0 @@ -export interface ExtractionConfig { - is_batch_event: boolean; - extraction_key: string; - dedup_config: DedupConfig; -} - -export interface DedupConfig { - drop_duplicates: boolean; - dedup_key: string; - dedup_period: number; -} - -export interface ValidationConfig { - validate: boolean; - mode: string; -} - -export interface DenormConfig { - redis_db_host: string; - redis_db_port: string; - denorm_fields: DenormFieldConfig; -} - -export interface DenormFieldConfig { - denorm_key: string; - redis_db: number; - denorm_out_field: string; -} - -export interface RouterConfig { - topic: string; -} - -export interface DatasetConfig { - data_key: string; - timestamp_key: string; - exclude_fields: string[]; - entry_topic: string; - redis_db_host: string; - redis_db_port: number; - redis_db: number; - index_data: boolean; -} - diff --git a/api-service/src/v1/models/DatasetModels.ts b/api-service/src/v1/models/DatasetModels.ts deleted file mode 100644 index c2e5dde1..00000000 --- a/api-service/src/v1/models/DatasetModels.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { ValidationStatus } from "./ValidationModels"; -import { Request, Response } from "express"; - -export interface ISchemaGenerator { - generate: ((sample: Map) => any) | - ((sample: Map[]) => any); - process: ((sample: Map) => any) | - ((sample: Map[]) => any); -} -export interface IConnector { - connect(): any; - execute(sample: any, type?: any, topic?: any): any; - executeSql(sql: string[]): any; - close(): any -} - -// Interface with method for request body validation -export interface IValidator { - // Method to perform validation on the request body of a request - validate(data: any, id?: string): ValidationStatus | Promise; -} - -// Interface with method for request params validation -export interface QValidator extends IValidator { - // Method to perform validation on the query params of a request - validateQueryParams(data: any, id?: string): ValidationStatus | Promise; -} - -export interface Params { - status: string, - errmsg: string -} -export interface IResponse { - id: string, - ts: number, - ver: string, - params: Params, - responseCode: string, - result: any -} - -export interface Result { - data: object; - status: number; -} - -export enum DatasetStatus { - Live = 'Live', Retired = 'Retired', -} - -export enum TransformationMode { - Strict = 'Strict', Lenient = 'Lenient', -} - -export enum ValidationMode { - Strict = 'Strict', IgnoreNewFields = 'IgnoreNewFields', DiscardNewFields = 'DiscardNewFields', -} diff --git a/api-service/src/v1/models/IngestionModels.ts b/api-service/src/v1/models/IngestionModels.ts deleted file mode 100644 index 3b8b9498..00000000 --- a/api-service/src/v1/models/IngestionModels.ts +++ /dev/null @@ -1,35 +0,0 @@ -export interface IngestionSpecModel { - dimensions: any, - metrics: any, - flattenSpec: any -} - -export interface IOConfig { - topic: string, - bootstrapIp: string, - taskDuration: string, - completionTimeout: string -} -export interface TuningConfig { - maxRowPerSegment: number, - taskCount: number, -} - -export interface GranularitySpec { - segmentGranularity: string, - queryGranularity: string, - rollup: boolean -} - -export interface IngestionConfig { - dataset: string, - indexCol: string, - granularitySpec: GranularitySpec, - tuningConfig?: TuningConfig, - ioConfig?: IOConfig -} - -export interface IngestionSchemeRequest { - schema: Map[], - config: IngestionConfig -} diff --git a/api-service/src/v1/resources/Constants.json b/api-service/src/v1/resources/Constants.json deleted file mode 100644 index 5bb0f303..00000000 --- a/api-service/src/v1/resources/Constants.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "ERROR_MESSAGE": { - "INVALID_DATE_RANGE": "Invalid date range! make sure your range must be valid and cannot be more than ${allowedRange} days", - "NO_DATE_RANGE": "Invalid date range! the date range cannot be a null value", - "ROUTE_NOT_FOUND": "Route not found", - "INVALID_DATA_SOURCE": "limits for datasource does not exist!" - }, - "STATUS": { - "SUCCESS": "SUCCESS", - "FAILURE": "FAILED" - }, - "DATASET": { - "CREATED": "The data has been successfully ingested" - }, - "CONFIG": { - "DATASET_SAVED": "The dataset configuration has been saved successfully", - "DATASET_UPDATED": "The dataset configuration has been updated successfully", - "DATASOURCE_SAVED": "The datasource configuration has been saved successfully", - "DATASOURCE_UPDATED": "The datasource configuration has been updated successfully", - "DATASET_PUBLISHED": "The dataset configuration has been published successfully", - "DATASOURCE_PUBLISHED": "The datasource configuration has been published successfully", - "DATASET_RETIRED": "The dataset configuration has been retired successfully" - }, - "RECORD_NOT_FOUND": { - "message": "Record not found", - "status": 404, - "code": "NOT_FOUND" - }, - "DATASET_ID_NOT_FOUND": { - "message": "Dataset Id not found, Failed to ingest", - "status": 404, - "code": "NOT_FOUND" - }, - "DUPLICATE_RECORD": { - "message": "Record already Exists", - "status": 409, - "code": "CONFLICT" - }, - "FAILED_RECORD_UPDATE": { - "message": "Failed to save Record", - "status": 400, - "code": "BAD_REQUEST" - }, - "FAILED_RECORD_CREATE": { - "message": "Failed to save Record", - "status": 400, - "code": "BAD_REQUEST" - }, - "FAILED_RECORD_FETCH": { - "message": "Failed to fetch Records", - "status": 400, - "code": "BAD_REQUEST" - }, - "FAILED_SQL_QUERY": { - "message": "Failed to execute the request", - "status": 400, - "code": "BAD_REQUEST" - }, - "EMPTY_DATASET_ID": { - "message": "datasetId parameter in url cannot be empty", - "status": 400, - "code": "BAD_REQUEST" - }, - "INVALID_DATASET_CONFIG": { - "message": "Invalid dataset config provided for ingestion", - "status": 400, - "code": "BAD_REQUEST" - }, - "RECORD_SAVED": "The Record has been saved successfully", - "RECORD_UPDATED": "The Record has been update successfully", - "TABLE_NOT_FOUND": { - "message": "Table does not exist in database", - "status": 404, - "code": "NOT_FOUND" - }, - "EXHAUST": { - "NO_BACKUP_FILES": "Date range provided does not have any backup files" - }, - "INVALID_DATASOURCE_REF": { - "message": "Datasource ref must match ingestion spec datasource", - "status": 400, - "code": "BAD_REQUEST" - }, - "INVALID_TOPIC": { - "message": "topic name in ingestion spec must match dataset id", - "status": 400, - "code": "BAD_REQUEST" - }, - "DATASET_NOT_FOUND": { - "message": "Dataset id does not exist in database", - "status": 404, - "code": "NOT_FOUND" - }, - "INVALID_DATASOURCE": { - "message": "Datasource ${datasource} not available for querying", - "status": 404, - "code": "NOT_FOUND" - }, - "DUPLICATE_DENORM_FIELD": { - "message": "Duplicate found for denorm output key", - "status": 400, - "code": "BAD_REQUEST" - }, - "INGESTION_SUBMITTED": "ingestion spec has been submitted successfully", - "INGESTION_FAILED_ON_SAVE": "Failed to submit Ingestion Spec, record is not saved", - "FAILED_EXECUTING_QUERY": { - "status": 500, - "message": "Something went wrong while executing the query. Please try again later.", - "code": "INTERNAL_SERVER_ERROR" -} -} diff --git a/api-service/src/v1/resources/schemas/DataExhaustReq.json b/api-service/src/v1/resources/schemas/DataExhaustReq.json deleted file mode 100644 index 44b947cb..00000000 --- a/api-service/src/v1/resources/schemas/DataExhaustReq.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "oneOf": [ - { - "type": "object", - "properties": { - "type": { "type": "string" }, - "from": { "type": "string", "format": "date" }, - "to": { "type": "string", "format": "date" } - }, - "required": ["from", "to", "type"] - }, - { - "type": "object", - "properties": { - "type": { "type": "string" }, - "since": { "type": "string", "pattern": "^\\w{4}_\\d{1,2}_\\w{4}" } - }, - "required": ["since", "type"] - } - ] -} diff --git a/api-service/src/v1/resources/schemas/DataIngestionReq.json b/api-service/src/v1/resources/schemas/DataIngestionReq.json deleted file mode 100644 index 060c1b95..00000000 --- a/api-service/src/v1/resources/schemas/DataIngestionReq.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "object", - "properties": { - "data": { - "type": "object" - } - }, - "required": [ - "data" - ] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetConfigDefault.ts b/api-service/src/v1/resources/schemas/DatasetConfigDefault.ts deleted file mode 100644 index f5ba0576..00000000 --- a/api-service/src/v1/resources/schemas/DatasetConfigDefault.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { config } from "../../configs/Config"; -import { DatasetStatus, ValidationMode } from "../../models/DatasetModels"; - -export const defaultConfig = { - "master": { - "dataset_config": { - "data_key": "", - "timestamp_key": "", - "exclude_fields": [], - "entry_topic": config.telemetry_service_config.kafka.topics.createMasterDataset, - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "index_data": true, - "redis_db": 3 - }, - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": false, - "dedup_key": "", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "router_config": { - "topic": "" - }, - "tags": [], - "status": DatasetStatus.Live, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" - }, - "dataset": { - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "router_config": { - "topic": "" - }, - "tags": [], - "dataset_config": { - "data_key": "", - "timestamp_key": "", - "exclude_fields": [], - "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "index_data": true, - "redis_db": 0 // The default Redis database index. - }, - "status": DatasetStatus.Live, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" - }, - "sourceConfig": { - "connector_type": '', - "connector_config": {}, - "status": DatasetStatus.Live, - "connector_stats": {}, - "created_by": 'SYSTEM', - "updated_by": 'SYSTEM' - } -} diff --git a/api-service/src/v1/resources/schemas/DatasetCreateReq.json b/api-service/src/v1/resources/schemas/DatasetCreateReq.json deleted file mode 100644 index fbabd4bc..00000000 --- a/api-service/src/v1/resources/schemas/DatasetCreateReq.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "extraction_config": { - "type": "object" - }, - "validation_config": { - "type": "object" - }, - "dedup_config": { - "type": "object" - }, - "data_schema": { - "type": "object" - }, - "denorm_config": { - "type": "object" - }, - "router_config": { - "type": "object", - "properties": { - "topic": { - "type": "string" - } - }, - "required": ["topic"] - }, - "dataset_config": { - "type": "object", - "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string" - }, - "exclude_fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "entry_topic": { - "type": "string" - }, - "redis_db_host": { - "type": "string" - }, - "redis_db_port": { - "type": "number" - }, - "redis_db": { - "type": "number" - }, - "index_data": { - "type": "boolean" - } - } - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "created_date": { - "type": "string" - }, - "updated_date": { - "type": "string" - }, - "published_date": { - "type": "string" - } - }, - "required": ["dataset_id", "router_config", "type", "published_date"] -} diff --git a/api-service/src/v1/resources/schemas/DatasetListReq.json b/api-service/src/v1/resources/schemas/DatasetListReq.json deleted file mode 100644 index 3cfc29d6..00000000 --- a/api-service/src/v1/resources/schemas/DatasetListReq.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "object", - "properties": { - "filters": { - "type": "object" - } - }, - "required": ["filters"] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json b/api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json deleted file mode 100644 index ed789d87..00000000 --- a/api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "connector_type": { - "type": "string" - }, - "connector_config": { - "type": "object" - }, - "status": { - "type": "string" - }, - "connector_stats": { - "type": "object" - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - } - }, - "required": [ - "connector_type", - "dataset_id" - ] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json b/api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json deleted file mode 100644 index ed789d87..00000000 --- a/api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "connector_type": { - "type": "string" - }, - "connector_config": { - "type": "object" - }, - "status": { - "type": "string" - }, - "connector_stats": { - "type": "object" - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - } - }, - "required": [ - "connector_type", - "dataset_id" - ] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetUpdateReq.json b/api-service/src/v1/resources/schemas/DatasetUpdateReq.json deleted file mode 100644 index bd8fcbb4..00000000 --- a/api-service/src/v1/resources/schemas/DatasetUpdateReq.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "extraction_config": { - "type": "object" - }, - "validation_config": { - "type": "object" - }, - "dedup_config": { - "type": "object" - }, - "data_schema": { - "type": "object" - }, - "denorm_config": { - "type": "object" - }, - "router_config": { - "type": "object", - "properties": { - "topic": { - "type": "string" - } - }, - "required": ["topic"] - }, - "dataset_config": { - "type": "object", - "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string" - }, - "exclude_fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "entry_topic": { - "type": "string" - }, - "redis_db_host": { - "type": "string" - }, - "redis_db_port": { - "type": "number" - }, - "redis_db": { - "type": "number" - }, - "index_data": { - "type": "boolean" - } - } - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "created_date": { - "type": "string" - }, - "updated_date": { - "type": "string" - }, - "published_date": { - "type": "string" - } - }, - "required": ["dataset_id"] -} diff --git a/api-service/src/v1/resources/schemas/DatasourceConfigDefault.json b/api-service/src/v1/resources/schemas/DatasourceConfigDefault.json deleted file mode 100644 index 48645bfd..00000000 --- a/api-service/src/v1/resources/schemas/DatasourceConfigDefault.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "retention_period": { - "enabled": "false" - }, - "archival_policy": { - "enabled": "false" - }, - "purge_policy": { - "enabled": "false" - }, - "backup_config": { - "enabled": "false" - }, - "status": "Live", - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "metadata": { - "aggregated": false, - "granularity": "day" - }, - "type": "druid" -} diff --git a/api-service/src/v1/resources/schemas/DatasourceSaveReq.json b/api-service/src/v1/resources/schemas/DatasourceSaveReq.json deleted file mode 100644 index 952bfaea..00000000 --- a/api-service/src/v1/resources/schemas/DatasourceSaveReq.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "type": "object", - "properties": { - "datasource": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string", - "enum": ["druid", "datalake"] - }, - "datasource_ref": { - "type": "string" - }, - "retention_period": { - "type": "object" - }, - "archival_policy": { - "type": "object" - }, - "purge_policy": { - "type": "object" - }, - "backup_config": { - "type": "object" - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "published_date": { - "type": "string" - }, - "metadata": { - "type": "object", - "properties": { - "aggregated": { - "type": "boolean" - }, - "granularity": { - "type": "string" - } - } - }, - "ingestion_spec": { - "type": "object" - } - }, - "required": ["dataset_id", "datasource", "published_date"], - - - "if": { - "properties": { - "type": { - "const": "druid" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object" - } - } - }, - "else": { - "if": { - "properties": { - "type": { - "const": "datalake" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object", - "properties": { - "dataset": { - "type": "string" - }, - "schema": { - "type": "object", - "properties": { - "partitionColumn": { - "type": "string" - }, - "primaryKey": { - "type": "string" - }, - "timestampColumn": { - "type": "string" - } - }, - "required": [ - "partitionColumn", - "primaryKey", - "timestampColumn" - ] - } - }, - "required": [ - "dataset", - "schema" - ] - } - } - }, - "else": { - "properties": { - "type": { - "enum": [ - "datalake", - "druid" - ] - } - } - } - } -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasourceUpdateReq.json b/api-service/src/v1/resources/schemas/DatasourceUpdateReq.json deleted file mode 100644 index c60a8066..00000000 --- a/api-service/src/v1/resources/schemas/DatasourceUpdateReq.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "type": "object", - "properties": { - "datasource": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string", - "enum": ["druid", "datalake"] - }, - "datasource_ref": { - "type": "string" - }, - "retention_period": { - "type": "object" - }, - "archival_policy": { - "type": "object" - }, - "purge_policy": { - "type": "object" - }, - "backup_config": { - "type": "object" - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "published_date": { - "type": "string" - }, - "metadata": { - "type": "object", - "properties": { - "aggregated": { - "type": "boolean" - }, - "granularity": { - "type": "string" - } - } - }, - "ingestion_spec": { - "type": "object" - } - }, - "required": ["dataset_id", "datasource"], - - "if": { - "properties": { - "type": { - "const": "druid" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object" - } - } - }, - "else": { - "if": { - "properties": { - "type": { - "const": "datalake" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object", - "properties": { - "dataset": { - "type": "string" - }, - "schema": { - "type": "object", - "properties": { - "table": { - "type": "string" - }, - "partitionColumn": { - "type": "string" - }, - "primaryKey": { - "type": "string" - }, - "timestampColumn": { - "type": "string" - } - }, - "required": [ - "partitionColumn", - "primaryKey", - "timestampColumn" - ] - } - }, - "required": [ - "dataset", - "schema" - ] - } - } - }, - "else": { - "properties": { - "type": { - "enum": [ - "datalake", - "druid" - ] - } - } - } - } -} diff --git a/api-service/src/v1/resources/schemas/QueryRequest.json b/api-service/src/v1/resources/schemas/QueryRequest.json deleted file mode 100644 index 9416c851..00000000 --- a/api-service/src/v1/resources/schemas/QueryRequest.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "query": { - "type": "object", - "nullable": true - }, - "querySql": { - "type": "object", - "properties": { - "query": { - "type": "string" - } - }, - "required": [ - "query" - ], - "nullable": true - }, - "context": { - "type": "object", - "properties": { - "dataSource": { - "type": "string" - }, - "granularity": { - "type": "string", - "enum": ["five_minute", "ten_minute", "fifteen_minute", "thirty_minute", "hour", "six_hour", "eight_hour", "day", "week", "month", "quarter", "year"] - }, - "dataSourceType": { - "type": "string", - "enum": ["realtime", "datalake"] - } - } - } - }, - "oneOf": [ - { - "required": [ - "querySql" - ] - }, - { - "required": [ - "query" - ] - } - ] -} diff --git a/api-service/src/v1/resources/schemas/SchemaValidatorReq.json b/api-service/src/v1/resources/schemas/SchemaValidatorReq.json deleted file mode 100644 index 50dbe8af..00000000 --- a/api-service/src/v1/resources/schemas/SchemaValidatorReq.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "filters": { - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "minLength": 3 - } - }, - "required": [ - "dataset_id" - ] - }, - "isLive": { - "type": "boolean" - }, - "event": { - "type": "object" - }, - "additionalProperties": false - }, - "required": [ - "filters", - "isLive", - "event" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/SubmitIngestionReq.json b/api-service/src/v1/resources/schemas/SubmitIngestionReq.json deleted file mode 100644 index 7c8020bc..00000000 --- a/api-service/src/v1/resources/schemas/SubmitIngestionReq.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "spec": { - "type": "object" - } - }, - "required": ["type", "spec"] -} diff --git a/api-service/src/v1/routes/Router.ts b/api-service/src/v1/routes/Router.ts deleted file mode 100644 index 2da73a9c..00000000 --- a/api-service/src/v1/routes/Router.ts +++ /dev/null @@ -1,80 +0,0 @@ -import express from "express"; -import { config } from "../configs/Config"; -import { HTTPConnector } from "../connectors/HttpConnector"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { QueryService } from "../services/QueryService"; -import { ValidationService } from "../services/ValidationService"; -import { DatasetService } from "../services/DatasetService"; -import { KafkaConnector } from "../connectors/KafkaConnector"; -import { DataSourceService } from "../services/DataSourceService"; -import { DatasetSourceConfigService } from "../services/DatasetSourceConfigService"; -import { DbConnector } from "../connectors/DbConnector"; -import { routesConfig } from "../configs/RoutesConfig"; -import { IngestorService } from "../services/IngestorService"; -import { OperationType, telemetryAuditStart } from "../services/telemetry"; -import telemetryActions from "../data/telemetryActions"; -import { ClientCloudService } from "../services/ClientCloudService"; -import { WrapperService } from "../services/WrapperService"; -import { onRequest } from "../../v2/metrics/prometheus/helpers" -import { Entity } from "../../v2/types/MetricModel"; -import { HealthService } from "../services/HealthService"; -import { eventsValidationAgainstSchema } from "../services/EventsValidationAgainstSchema"; - -export const validationService = new ValidationService(); -const httpDruidConnector = new HTTPConnector(`${config.query_api.druid.host}:${config.query_api.druid.port}`) -export const queryService = new QueryService(httpDruidConnector); -export const kafkaConnector = new KafkaConnector() -export const dbConnector = new DbConnector(config.db_connector_config); -export const datasourceService = new DataSourceService(dbConnector, config.table_names.datasources); -export const datasetService = new DatasetService(dbConnector, config.table_names.datasets); -export const datasetSourceConfigService = new DatasetSourceConfigService(dbConnector, config.table_names.datasetSourceConfig); -export const ingestorService = new IngestorService(kafkaConnector,); -export const exhaustService = new ClientCloudService(config.exhaust_config.cloud_storage_provider, config.exhaust_config.cloud_storage_config); -export const wrapperService = new WrapperService(); -export const globalCache: any = new Map() -export const healthService = new HealthService(dbConnector, kafkaConnector, httpDruidConnector) -export const router = express.Router() -dbConnector.init() -/** Query API(s) */ - -router.post([`${routesConfig.query.native_query.path}`, `${routesConfig.query.native_query_with_params.path}`,], ResponseHandler.setApiId(routesConfig.query.native_query.api_id), telemetryAuditStart({ action: telemetryActions.nativeQuery, operationType: OperationType.GET }), onRequest({ entity: Entity.Data_out }), validationService.validateRequestBody, validationService.validateQuery, queryService.executeNativeQuery); -router.post([`${routesConfig.query.sql_query.path}`, `${routesConfig.query.sql_query_with_params.path}`,], ResponseHandler.setApiId(routesConfig.query.sql_query.api_id), telemetryAuditStart({ action: telemetryActions.sqlQuery, operationType: OperationType.GET }), onRequest({ entity: Entity.Data_out }), validationService.validateRequestBody, validationService.validateQuery, queryService.executeSqlQuery); - -/** Ingestor API */ -router.post(`${routesConfig.data_ingest.path}`, ResponseHandler.setApiId(routesConfig.data_ingest.api_id), telemetryAuditStart({ action: telemetryActions.ingestEvents, operationType: OperationType.CREATE }), onRequest({ entity: Entity.Data_in }), validationService.validateRequestBody, ingestorService.create); -router.post(`${routesConfig.tenant_ingest.path}`, ResponseHandler.setApiId(routesConfig.tenant_ingest.api_id), telemetryAuditStart({ action: telemetryActions.ingestEvents, operationType: OperationType.CREATE }), onRequest({ entity: Entity.Data_in }), validationService.validateRequestBody, ingestorService.tenant); - -/** Dataset APIs */ -router.post(`${routesConfig.config.dataset.save.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.save.api_id), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasetService.save); -router.patch(`${routesConfig.config.dataset.update.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.update.api_id), telemetryAuditStart({ action: telemetryActions.updateDataset, operationType: OperationType.UPDATE }), validationService.validateRequestBody, datasetService.update); -router.get(`${routesConfig.config.dataset.read.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.read.api_id), telemetryAuditStart({ action: telemetryActions.readDataset, operationType: OperationType.GET }), datasetService.read); -router.post(`${routesConfig.config.dataset.list.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.list.api_id), telemetryAuditStart({ action: telemetryActions.listDatasets, operationType: OperationType.LIST }), validationService.validateRequestBody, datasetService.list); - -/** Dataset Source Config APIs */ -router.post(`${routesConfig.config.dataset_source_config.save.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.save.api_id), telemetryAuditStart({ action: telemetryActions.createDatasetSourceConfig, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasetSourceConfigService.save); -router.patch(`${routesConfig.config.dataset_source_config.update.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.update.api_id), telemetryAuditStart({ action: telemetryActions.updateDatasetSourceConfig, operationType: OperationType.UPDATE }), validationService.validateRequestBody, datasetSourceConfigService.update); -router.get(`${routesConfig.config.dataset_source_config.read.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.read.api_id), telemetryAuditStart({ action: telemetryActions.getatasetSourceConfig, operationType: OperationType.GET }), datasetSourceConfigService.read); -router.post(`${routesConfig.config.dataset_source_config.list.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.list.api_id), telemetryAuditStart({ action: telemetryActions.listDatasetSourceConfig, operationType: OperationType.LIST }), validationService.validateRequestBody, datasetSourceConfigService.list); - -/** DataSource API(s) */ -router.post(`${routesConfig.config.datasource.save.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.save.api_id), telemetryAuditStart({ action: telemetryActions.createDatasource, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasourceService.save); -router.patch(`${routesConfig.config.datasource.update.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.update.api_id), telemetryAuditStart({ action: telemetryActions.updateDatasource, operationType: OperationType.UPDATE }), validationService.validateRequestBody, datasourceService.update); -router.get(`${routesConfig.config.datasource.read.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.read.api_id), telemetryAuditStart({ action: telemetryActions.getDatasource, operationType: OperationType.GET }), datasourceService.read); -router.post(`${routesConfig.config.datasource.list.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.list.api_id), telemetryAuditStart({ action: telemetryActions.listDatasource, operationType: OperationType.LIST }), validationService.validateRequestBody, datasourceService.list); - -/** Exhaust API(s) */ -router.get(`${routesConfig.exhaust.path}`, ResponseHandler.setApiId(routesConfig.exhaust.api_id), telemetryAuditStart({ action: telemetryActions.datasetExhaust, operationType: OperationType.GET }), validationService.validateRequestParams, exhaustService.getData); - -/*** Submit Ingestion API(s) */ -router.post(`${routesConfig.submit_ingestion.path}`, ResponseHandler.setApiId(routesConfig.submit_ingestion.api_id), telemetryAuditStart({ action: telemetryActions.submitIngestionSpec, operationType: OperationType.CREATE }), validationService.validateRequestBody, ingestorService.submitIngestion) - -/** Query Wrapper API(s) */ -router.post(routesConfig.query_wrapper.sql_wrapper.path, ResponseHandler.setApiId(routesConfig.query_wrapper.sql_wrapper.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardSql) -router.post(routesConfig.query_wrapper.native_post.path, ResponseHandler.setApiId(routesConfig.query_wrapper.native_post.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardNative) -router.get(routesConfig.query_wrapper.native_get.path, ResponseHandler.setApiId(routesConfig.query_wrapper.native_get.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardNativeGet) -router.delete(routesConfig.query_wrapper.native_delete.path, ResponseHandler.setApiId(routesConfig.query_wrapper.native_delete.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardNativeDel) -router.get(routesConfig.query_wrapper.druid_status.path, ResponseHandler.setApiId(routesConfig.query_wrapper.druid_status.api_id), wrapperService.nativeStatus) -router.get(routesConfig.health.path, ResponseHandler.setApiId(routesConfig.health.api_id), healthService.checkHealth.bind(healthService)) - -/* schema validator */ -router.post(`${routesConfig.schema_validator.path}`, ResponseHandler.setApiId(routesConfig.schema_validator.api_id), validationService.validateRequestBody, eventsValidationAgainstSchema) diff --git a/api-service/src/v1/services/ClientCloudService.ts b/api-service/src/v1/services/ClientCloudService.ts deleted file mode 100644 index 3b588462..00000000 --- a/api-service/src/v1/services/ClientCloudService.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { NextFunction, Request, Response } from 'express'; -import { ResponseHandler } from '../helpers/ResponseHandler'; -import httpStatus from 'http-status'; -import constants from '../resources/Constants.json'; -import { ingestorService } from '../routes/Router'; -import { getDateRange, isValidDateRange } from '../utils/common'; -import { config as globalConfig } from '../configs/Config'; -import CloudService from '../lib/client-cloud-services'; -import moment from "moment"; -import { DateRange } from '../models/ExhaustModels'; -import { updateTelemetryAuditEvent } from './telemetry'; - -export class ClientCloudService { - private cloudProvider: string - private client: any - private storage: any - private config: any - private momentFormat: string; - constructor(cloudProvider: string, config?: any) { - this.cloudProvider = cloudProvider - this.client = CloudService.init(this.cloudProvider) - this.config = config - this.storage = new this.client(this.config) - this.momentFormat = "YYYY-MM-DD"; - } - - verifyDatasetExists = async (datasetId: string) => { - const datasetRecord = await ingestorService.getDatasetConfig(datasetId); - return datasetRecord; - } - - getFromStorage = async (type: string | undefined, dateRange: DateRange, datasetId: string) => { - let resData: any = {}; - resData = await this.storage.getFiles( - globalConfig.exhaust_config.container, globalConfig.exhaust_config.container_prefix, type, dateRange, datasetId, - ) - return resData; - } - - getData = async (req: Request, res: Response, next: NextFunction) => { - const { params } = req; - const { datasetId } = params; - const { type } = req.query; - updateTelemetryAuditEvent({ request: req, object: { id: datasetId, type: "dataset", ver: "1.0.0" } }); - // Validations - if(type && globalConfig.exhaust_config.exclude_exhaust_types.includes(type.toString())) { - next({statusCode: 404, message: constants.RECORD_NOT_FOUND, errCode: httpStatus["404_NAME"],}) - return; - } - const datasetRecord = await this.verifyDatasetExists(datasetId); - if(!datasetRecord) { - next({statusCode: 404, message: constants.RECORD_NOT_FOUND, errCode: httpStatus["404_NAME"],}) - return; - } - const dateRange = getDateRange(req, res); - const isValidDates = isValidDateRange( - moment(dateRange.from, this.momentFormat), - moment(dateRange.to, this.momentFormat), - globalConfig.exhaust_config.maxQueryDateRange, - ); - if(!isValidDates) { - next({statusCode: 400, message: constants.ERROR_MESSAGE.INVALID_DATE_RANGE.replace("${allowedRange}", globalConfig.exhaust_config.maxQueryDateRange.toString()), errCode: httpStatus["400_NAME"],}) - return; - } - - // Fetch Data - const resData: any = await this.getFromStorage(type?.toString(), dateRange, datasetId); - if(resData.files.length === 0) { - next({statusCode: 404, message: constants.EXHAUST.NO_BACKUP_FILES, errCode: httpStatus["404_NAME"],}) - return; - } - ResponseHandler.successResponse(req, res, { status: 200, data: resData, }) - } -} diff --git a/api-service/src/v1/services/DataSourceService.ts b/api-service/src/v1/services/DataSourceService.ts deleted file mode 100644 index 1942535a..00000000 --- a/api-service/src/v1/services/DataSourceService.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import _ from 'lodash' -import { Datasources } from "../helpers/Datasources"; -import { findAndSetExistingRecord, updateTelemetryAuditEvent } from "./telemetry"; -import { DbUtil } from "../helpers/DbUtil"; -import constants from "../resources/Constants.json"; -import { ingestorService } from "../routes/Router"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; -import { config } from "../configs/Config"; - -const telemetryObject = { id: null, type: "datasource", ver: "1.0.0" }; - -export class DataSourceService { - private table: string - private dbConnector: IConnector; - private dbUtil: DbUtil - private errorHandler: ErrorResponseHandler; - - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - this.dbUtil = new DbUtil(dbConnector, table) - this.errorHandler = new ErrorResponseHandler("DataSourceService"); - } - - public save = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasources = new Datasources(req.body) - const payload: any = datasources.setValues() - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(payload, 'id'), } }); - if(payload.type === config.datasource_storage_types.druid) await this.validateDruidDatasource(payload) - else this.validateLakehouseDatasource(payload) - await this.dbUtil.save(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public update = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasources = new Datasources(req.body) - const payload: Record = datasources.setValues() - if(payload.type === config.datasource_storage_types.druid) await this.validateDruidDatasource(payload) - else this.validateLakehouseDatasource(payload) - await findAndSetExistingRecord({ dbConnector: this.dbConnector, table: this.table, request: req, filters: { "id": _.get(payload, 'id') }, object: { ...telemetryObject, id: _.get(payload, 'id') } }); - await this.dbUtil.upsert(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public read = async (req: Request, res: Response, next: NextFunction) => { - try { - let status: any = req.query.status || DatasetStatus.Live - const id = req.params.datasourceId - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id } }); - await this.dbUtil.read(req, res, next, { id, status }) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - public list = async (req: Request, res: Response, next: NextFunction) => { - try { - const payload = req.body - await this.dbUtil.list(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - public validateDruidDatasource = async (payload: Record) => { - let datasetRecord = await ingestorService.getDatasetConfig(payload.dataset_id); - if (_.isEmpty(datasetRecord)) { - throw constants.DATASET_NOT_FOUND; - } - if (!_.isUndefined(payload.datasource_ref) && !_.isUndefined(payload.ingestion_spec?.spec?.dataSchema?.dataSource) && payload.datasource_ref !== payload.ingestion_spec?.spec?.dataSchema?.dataSource) { - throw constants.INVALID_DATASOURCE_REF; - } - if ( - !_.isUndefined(payload.dataset_id) && !_.isUndefined(payload.ingestion_spec) && datasetRecord.router_config.topic !== payload.ingestion_spec?.spec?.ioConfig?.topic) { - throw constants.INVALID_TOPIC; - } - } - - public validateLakehouseDatasource = (payload: Record) => { - if (_.isUndefined(payload.ingestion_spec.schema.table) || _.isEmpty(payload.ingestion_spec.schema.table)) { - payload.ingestion_spec.schema.table = payload.datasource_ref - } - payload.ingestion_spec.schema.table = payload.ingestion_spec.schema.table.replace(/-/g, '_') - } -} diff --git a/api-service/src/v1/services/DatasetService.ts b/api-service/src/v1/services/DatasetService.ts deleted file mode 100644 index 151db15d..00000000 --- a/api-service/src/v1/services/DatasetService.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import _ from 'lodash' -import { Datasets } from "../helpers/Datasets"; -import { findAndSetExistingRecord, updateTelemetryAuditEvent } from "./telemetry"; -import { DbUtil } from "../helpers/DbUtil"; -import { refreshDatasetConfigs } from "../helpers/DatasetConfigs"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; - -const telemetryObject = { id: null, type: "dataset", ver: "1.0.0" }; - -export class DatasetService { - private table: string - private dbConnector: IConnector; - private dbUtil: DbUtil; - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - this.dbUtil = new DbUtil(dbConnector, table) - this.errorHandler = new ErrorResponseHandler("DatasetService"); - } - - public save = async (req: Request, res: Response, next: NextFunction) => { - try { - const dataset = new Datasets(req.body) - const payload: any = dataset.setValues() - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(payload, 'dataset_id') } }); - await this.dbUtil.save(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - - public update = async (req: Request, res: Response, next: NextFunction) => { - try { - const dataset = new Datasets(req.body) - const payload = dataset.getValues() - await findAndSetExistingRecord({ dbConnector: this.dbConnector, table: this.table, request: req, filters: { "id": payload.id }, object: { ...telemetryObject, id: payload.id } }); - await this.dbUtil.update(req, res, next, payload) - await refreshDatasetConfigs() - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - - public read = async (req: Request, res: Response, next: NextFunction) => { - try { - let status: any = req.query.status || DatasetStatus.Live - const id = req.params.datasetId - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id } }); - await this.dbUtil.read(req, res, next, { id, status }) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - - public list = async (req: Request, res: Response, next: NextFunction) => { - try { - const payload = req.body - await this.dbUtil.list(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } -} diff --git a/api-service/src/v1/services/DatasetSourceConfigService.ts b/api-service/src/v1/services/DatasetSourceConfigService.ts deleted file mode 100644 index 0f19dd50..00000000 --- a/api-service/src/v1/services/DatasetSourceConfigService.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import _ from 'lodash' -import { DatasetSourceConfigs } from "../helpers/DatasetSourceConfigs"; -import { DbUtil } from "../helpers/DbUtil"; -import { findAndSetExistingRecord, updateTelemetryAuditEvent } from "./telemetry"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; - -const telemetryObject = { id: null, type: "dataset-source-config", ver: "1.0.0" }; - -export class DatasetSourceConfigService { - private table: string - private dbConnector: IConnector; - private dbUtil: DbUtil - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - this.dbUtil = new DbUtil(dbConnector, table) - this.errorHandler = new ErrorResponseHandler("DatasetSourceConfigService"); - } - - public save = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasetSourceConfig = new DatasetSourceConfigs(req.body) - const payload: any = datasetSourceConfig.setValues() - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(payload, 'id') } }); - await this.dbUtil.save(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public update = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasetSourceConfig = new DatasetSourceConfigs(req.body) - const payload: Record = datasetSourceConfig.setValues() - await findAndSetExistingRecord({ dbConnector: this.dbConnector, table: this.table, request: req, filters: { "id": _.get(payload, 'id') }, object: { ...telemetryObject, id: _.get(payload, 'id') } }); - await this.dbUtil.upsert(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public read = async (req: Request, res: Response, next: NextFunction) => { - try { - let status: any = req.query.status || DatasetStatus.Live - const id = req.params.datasetId - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id } }); - await this.dbUtil.read(req, res, next, { id, status }) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - public list = async (req: Request, res: Response, next: NextFunction) => { - try { - const payload = req.body - await this.dbUtil.list(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } -} diff --git a/api-service/src/v1/services/EventsValidationAgainstSchema.ts b/api-service/src/v1/services/EventsValidationAgainstSchema.ts deleted file mode 100644 index b25e34f4..00000000 --- a/api-service/src/v1/services/EventsValidationAgainstSchema.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import * as _ from "lodash" -import { schemaValidation } from "../helpers/ValidationService"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { dbConnector } from "../routes/Router"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { config } from "../configs/Config"; - -export const eventsValidationAgainstSchema = async (req: Request, res: Response, next: NextFunction) => { - const errorHandler = new ErrorResponseHandler("DatasetService"); - try { - const isLive = _.get(req, "body.isLive"); - const event = _.get(req, "body.event"); - const filters = _.get(req, "body.filters") - let datasetRecord = await dbConnector.readRecords(isLive ? config.table_names.datasets : `${config.table_names.datasets}_draft`, { filters: filters }) - let schema = _.get(datasetRecord, "[0].data_schema") - - if (_.isEmpty(datasetRecord)) { - throw { - "message": `Dataset ${filters?.dataset_id} does not exists`, - "status": 404, - "code": "NOT_FOUND" - } - } - - const validateEventAgainstSchema = schemaValidation(event, _.omit(schema, "$schema")); - ResponseHandler.successResponse(req, res, { status: 200, data: { message: validateEventAgainstSchema?.message, isValid: validateEventAgainstSchema?.isValid } }); - } - catch (error) { - return errorHandler.handleError(req, res, next, error); - } -} \ No newline at end of file diff --git a/api-service/src/v1/services/HealthService.ts b/api-service/src/v1/services/HealthService.ts deleted file mode 100644 index 6a79ede9..00000000 --- a/api-service/src/v1/services/HealthService.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { AxiosInstance } from "axios"; -import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { IConnector } from "../models/DatasetModels"; -import { DbConnector } from "../connectors/DbConnector"; -import { KafkaConnector } from "../connectors/KafkaConnector"; - - - - -export class HealthService { - private dbConnector: DbConnector; - private kafkaConnector: KafkaConnector; - private httpDruidConnector: AxiosInstance; - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: DbConnector, kafkaConnector: KafkaConnector, httpDruidConnector: IConnector) { - this.errorHandler = new ErrorResponseHandler("HealthService"); - this.httpDruidConnector = httpDruidConnector.connect() - this.dbConnector = dbConnector; - this.kafkaConnector = kafkaConnector; - } - - checkHealth(req: Request, res: Response, next: NextFunction) { - Promise.all([this.checkDruidHealth(), this.checkKafkaHealth(), this.checkPostgresHealth()]) - .then(() => { - ResponseHandler.successResponse(req, res, { status: 200, data: {} }) - }).catch(error => { - this.errorHandler.handleError(req, res, next, error) - }) - } - - private async checkDruidHealth() { - await this.httpDruidConnector.get("/status/health") - } - - private async checkKafkaHealth() { - await this.kafkaConnector.connect() - } - - private async checkPostgresHealth() { - await this.dbConnector.health() - } - -} diff --git a/api-service/src/v1/services/IngestorService.ts b/api-service/src/v1/services/IngestorService.ts deleted file mode 100644 index 482cd764..00000000 --- a/api-service/src/v1/services/IngestorService.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import constants from "../resources/Constants.json" -import { ResponseHandler } from "../helpers/ResponseHandler"; -import _ from 'lodash' -import { globalCache } from "../routes/Router"; -import { refreshDatasetConfigs } from "../helpers/DatasetConfigs"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; -import { wrapperService } from "../routes/Router"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -export class IngestorService { - private kafkaConnector: IConnector; - private errorHandler: ErrorResponseHandler; - constructor(kafkaConnector: IConnector,) { - this.kafkaConnector = kafkaConnector - this.errorHandler = new ErrorResponseHandler("IngestorService"); - this.init() - } - public init() { - this.kafkaConnector.connect() - .then(() => { - console.log("kafka connection arranged succesfully...") - }) - .catch((error: any) => { - console.log("error while connecting to kafka", error.message) - }) - } - - public create = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasetId = this.getDatasetId(req); - const validData = await this.validateData(req.body.data, datasetId); - req.body = { ...req.body.data, dataset: datasetId }; - const topic = await this.getTopic(datasetId); - await this.kafkaConnector.execute(req, res, topic); - ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.DATASET.CREATED } }); - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - - public tenant = async (req: Request, res: Response, next: NextFunction) => { - try { - let datasetId = this.getDatasetId(req); - const tenantId = _.get(req.headers, 'x-tenant-id', "default"); - datasetId = `${tenantId}-${datasetId}`; - const validData = await this.validateData(req.body.data, datasetId); - req.body = { ...req.body.data, dataset: datasetId }; - const topic = await this.getTopic(datasetId); - await this.kafkaConnector.execute(req, res, topic); - ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.DATASET.CREATED } }); - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - - public submitIngestion = async (req: Request, res: Response, next: NextFunction) => { - try { - await wrapperService.submitIngestion(req.body) - ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.INGESTION_SUBMITTED } }); - } - catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - private getDatasetId(req: Request) { - let datasetId = req.params.datasetId.trim() - if (!_.isEmpty(datasetId)) return datasetId - throw constants.EMPTY_DATASET_ID - } - - public async getDatasetConfig(datasetId: string) { - let datasetConfigList = globalCache.get("dataset-config"); - if (!datasetConfigList) await refreshDatasetConfigs(); - - datasetConfigList = globalCache.get("dataset-config"); - const datasetRecord = datasetConfigList.find((record: any) => record.id === datasetId && record.status === DatasetStatus.Live); - // Return record if present in cache - if (datasetRecord) return datasetRecord; - else { // Refresh dataset configs cache in case record present in cache - await refreshDatasetConfigs(); - const datasetConfigList = globalCache.get("dataset-config"); - const datasetRecord = datasetConfigList.find((record: any) => record.id === datasetId && record.status === DatasetStatus.Live); - return datasetRecord; - } - } - - private async getTopic(datasetId: string) { - const datasetRecord = await this.getDatasetConfig(datasetId); - if (!datasetRecord) throw constants.DATASET_ID_NOT_FOUND; - return datasetRecord.dataset_config.entry_topic; - } - - private async validateData(data: any, datasetId: string) { - const datasetRecord = await this.getDatasetConfig(datasetId); - if (!datasetRecord) throw constants.DATASET_ID_NOT_FOUND; - if(_.has(datasetRecord, "extraction_config") && _.get(datasetRecord, ["extraction_config", "is_batch_event"])) { - if( - _.has(data, _.get(datasetRecord, ["extraction_config", "extraction_key"])) && - _.has(data, _.get(datasetRecord, ["extraction_config", "batch_id"])) - ) - return data; - else if (_.has(data, "event")) - return data; - else throw constants.INVALID_DATASET_CONFIG; - } else { - if(_.has(data, "event")) - return data; - else throw constants.INVALID_DATASET_CONFIG; - } - } -} diff --git a/api-service/src/v1/services/QueryService.ts b/api-service/src/v1/services/QueryService.ts deleted file mode 100644 index f0b8706a..00000000 --- a/api-service/src/v1/services/QueryService.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { AxiosInstance } from "axios"; -import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; -import { config } from "../configs/Config"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { IConnector } from "../models/DatasetModels"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { updateTelemetryAuditEvent } from "./telemetry"; -import { executeLakehouseQuery } from "../helpers/LakehouseUtil"; - -const telemetryObject = { id: null, type: "datasource", ver: "1.0.0" }; - -export class QueryService { - private connector: AxiosInstance; - private errorHandler: ErrorResponseHandler; - constructor(connector: IConnector) { - this.connector = connector.connect(); - this.errorHandler = new ErrorResponseHandler("QueryService"); - } - - public executeNativeQuery = async (req: Request, res: Response, next: NextFunction) => { - try { - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(req, 'body.context.dataSource')} }); - var result = await this.connector.post(config.query_api.druid.native_query_path, req.body.query); - var mergedResult = result.data; - if (req.body.query.queryType === "scan" && result.data) { - mergedResult = result.data.map((item: Record) => { - return item.events; - }); - } - ResponseHandler.successResponse(req, res, { status: result.status, data: _.flatten(mergedResult) }); - - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } - }; - - public executeSqlQuery = async (req: Request, res: Response, next: NextFunction) => { - try { - let result: any = {} - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(req, 'body.context.dataSource')} }); - if (req.body?.context?.dataSourceType === config.query_api.lakehouse.queryType) { - result.data = await executeLakehouseQuery(req.body.querySql.query) - } - else{ - result = await this.connector.post(config.query_api.druid.sql_query_path, req.body.querySql); - } - ResponseHandler.successResponse(req, res, { status: result.status || 200, data: result.data }); - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } - } -} diff --git a/api-service/src/v1/services/ValidationService.ts b/api-service/src/v1/services/ValidationService.ts deleted file mode 100644 index 24865a19..00000000 --- a/api-service/src/v1/services/ValidationService.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { NextFunction, Request, Response } from "express"; -import httpStatus from "http-status"; -import { IValidator, QValidator } from "../models/DatasetModels"; -import { ValidationStatus } from "../models/ValidationModels"; -import { QueryValidator } from "../validators/QueryValidator"; -import { RequestsValidator } from "../validators/RequestsValidator"; - -export class ValidationService { - - private request: QValidator; - private query: IValidator; - constructor() { - this.request = new RequestsValidator() - this.query = new QueryValidator() - } - - public validateRequestBody = async (req: Request, res: Response, next: NextFunction) => { - const status: ValidationStatus = await this.request.validate(req.body, (req as any).id) - status.isValid ? - next() - : next({ statusCode: httpStatus.BAD_REQUEST, message: status.message || "", errCode: status.code }) - }; - - public validateRequestParams = async (req: Request, res: Response, next: NextFunction) => { - const status: ValidationStatus = await this.request.validateQueryParams(req.query, (req as any).id) - status.isValid ? - next() - : next({ statusCode: httpStatus.BAD_REQUEST, message: status.message || "", errCode: status.code }) - }; - - public validateQuery = async (req: Request, res: Response, next: NextFunction) => { - const status: ValidationStatus = await this.query.validate(req.body, (req as any).id) - status.isValid ? - next() - : next({ statusCode: httpStatus.BAD_REQUEST, message: status.message || "", errCode: status.code }) - }; -} diff --git a/api-service/src/v1/services/telemetry.ts b/api-service/src/v1/services/telemetry.ts deleted file mode 100644 index e63d1d09..00000000 --- a/api-service/src/v1/services/telemetry.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { Request, Response, NextFunction } from "express" -import { v4 } from 'uuid'; -import _ from 'lodash'; -import { config as appConfig } from "../configs/Config"; -import { Kafka } from 'kafkajs'; - -const env = _.get(appConfig, 'env') -const telemetryTopic = _.get(appConfig, 'telemetry_dataset'); -const brokerServers = _.get(appConfig, 'telemetry_service_config.kafka.config.brokers'); - -export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } - -const kafka = new Kafka({ clientId: telemetryTopic, brokers: brokerServers }); -const telemetryEventsProducer = kafka.producer(); -telemetryEventsProducer.connect().catch(err => console.error("Unable to connect to kafka", err.message)); - -const getDefaults = () => { - return { - eid: 'AUDIT', - ets: Date.now(), - ver: '1.0.0', - mid: v4(), - actor: { - id: "SYSTEM", - type: 'User' - }, - context: { - env, - sid: v4(), - pdata: { - id: `${env}.api.service`, - ver: '1.0.0' - } - }, - object: {}, - edata: {} - }; -}; - -const getDefaultEdata = ({ action }: any) => ({ - startEts: Date.now(), - type: null, - object: {}, - fromState: "inprogress", - toState: "completed", - edata: { - action, - props: [], - transition: { - timeUnit: "ms", - duration: 0 - } - } -}) - -const sendTelemetryEvents = async (event: Record) => { - telemetryEventsProducer.send({ topic: telemetryTopic, messages: [{ value: JSON.stringify(event) }] }).catch(console.log) -} - -const transformProps = (body: Record) => { - return _.map(_.entries(body), (payload) => { - const [key, value] = payload; - return { - property: key, - ov: null, - nv: value - } - }) -} - -export const setAuditState = (state: string, req: any) => { - if (state && req) { - _.set(req.auditEvent, "toState", state); - } -} - -const setAuditEventType = (operationType: any, request: any) => { - switch (operationType) { - case OperationType.CREATE: { - _.set(request, 'auditEvent.type', 'create'); - break; - } - case OperationType.UPDATE: { - _.set(request, 'auditEvent.type', 'update'); - break; - } - case OperationType.PUBLISH: { - _.set(request, 'auditEvent.type', 'publish'); - break; - } - case OperationType.RETIRE: { - _.set(request, 'auditEvent.type', 'retire'); - break; - } - case OperationType.LIST: { - _.set(request, 'auditEvent.type', 'list'); - break; - } - case OperationType.GET: { - _.set(request, 'auditEvent.type', 'get'); - break; - } - default: - break; - } -} - -export const telemetryAuditStart = ({ operationType, action }: any) => { - return async (request: any, response: Response, next: NextFunction) => { - try { - const body = request.body || {}; - request.auditEvent = getDefaultEdata({ action }); - const props = transformProps(body); - _.set(request, 'auditEvent.edata.props', props); - setAuditEventType(operationType, request); - } catch (error) { - console.log(error); - } finally { - next(); - } - } -} - -export const processAuditEvents = (request: Request, response: Response) => { - const auditEvent: any = _.get(request, 'auditEvent'); - if (auditEvent) { - const { startEts, object = {}, edata = {}, toState, fromState }: any = auditEvent; - const endEts = Date.now(); - const duration = startEts ? (endEts - startEts) : 0; - _.set(auditEvent, 'edata.transition.duration', duration); - if (toState && fromState) { - _.set(auditEvent, 'edata.transition.toState', toState); - _.set(auditEvent, 'edata.transition.fromState', fromState); - } - const telemetryEvent = getDefaults(); - _.set(telemetryEvent, 'edata', edata); - _.set(telemetryEvent, 'object', { ...(object.id && object.type && { ...object, ver: '1.0.0' }) }); - sendTelemetryEvents(telemetryEvent); - } -} - -export const interceptAuditEvents = () => { - return (request: Request, response: Response, next: NextFunction) => { - response.on('finish', () => { - const statusCode = _.get(response, 'statusCode'); - const isError = statusCode && statusCode >= 400; - !isError && processAuditEvents(request, response); - }) - next(); - } -} - -export const updateTelemetryAuditEvent = ({ currentRecord, request, object = {} }: Record) => { - const auditEvent = request?.auditEvent; - _.set(request, 'auditEvent.object', object); - if (currentRecord) { - const props = _.get(auditEvent, 'edata.props'); - const updatedProps = _.map(props, (prop: Record) => { - const { property, nv } = prop; - const existingValue = _.get(currentRecord, property); - return { property, ov: existingValue, nv }; - }); - _.set(request, 'auditEvent.edata.props', updatedProps); - } -} - -export const findAndSetExistingRecord = async ({ dbConnector, table, filters, request, object = {} }: Record) => { - const auditEvent = request?.auditEvent; - if (dbConnector && table && filters && _.get(auditEvent, 'type') === "update") { - try { - _.set(request, 'auditEvent.object', object); - const records = await dbConnector.execute("read", { table, fields: { filters } }) - const existingRecord = _.first(records); - if (existingRecord) { - const props = _.get(auditEvent, 'edata.props'); - const updatedProps = _.map(props, (prop: Record) => { - const { property, nv } = prop; - const existingValue = _.get(existingRecord, property); - return { property, ov: existingValue, nv }; - }); - _.set(request, 'auditEvent.edata.props', updatedProps); - } - } catch (error) { - setAuditState("failed", request); - } - } -} diff --git a/api-service/src/v1/test/ClientCloudService.spec.ts b/api-service/src/v1/test/ClientCloudService.spec.ts deleted file mode 100644 index 398ce60f..00000000 --- a/api-service/src/v1/test/ClientCloudService.spec.ts +++ /dev/null @@ -1,128 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import httpStatus from "http-status"; -import { TestExhaust } from "./Fixtures"; -import { config } from "./Config"; -import constants from "../resources/Constants.json"; -import { dbConnector, globalCache, exhaustService } from "../routes/Router"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; -import { config as appConfig } from "../configs/Config"; -import moment from "moment"; -import { DatasetStatus } from "../models/DatasetModels"; -chai.use(chaiSpies) -chai.should(); -chai.use(chaiHttp); - -describe("AWS Cloud Storage", () => { - beforeEach(() => { - chai.spy.on(globalCache, "get", () => { - return [{ "id": ":datasetId", "dataset_id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" } }] - }) - }) - - afterEach(() => { - chai.spy.restore() - }) - - it("it should return 404 if dataset record not found", (done) => { - chai.spy.restore() - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([{}]) - }) - chai.spy.on(globalCache, "get", () => { - return [] - }) - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.VALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.result.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]) - done() - }) - }); - - it("it should raise error when from or to have invalid dates", (done) => { - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.INVALID_DATE_RANGE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.result.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]) - done() - }) - }); - - it("it should raise error when time interval is greater than 31 days", (done) => { - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.DATE_RANGE_OVER_LIMIT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.params.errmsg.should.be.eq(constants.ERROR_MESSAGE.INVALID_DATE_RANGE.replace("${allowedRange}", `${config.exhaustMaxDateRange}`)) - res.body.result.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]) - done() - }) - }); - - it("it should return 404 when no files exist for given date range", (done) => { - chai.spy.on(exhaustService, "getFromStorage", () => { - return Promise.resolve({ - expiresAt: moment().add(appConfig.exhaust_config.storage_url_expiry, "seconds").toISOString(), - files: [], - periodWiseFiles: {}, - }); - }) - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.VALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.result.should.be.a("object") - res.body.params.errmsg.should.be.eq(constants.EXHAUST.NO_BACKUP_FILES) - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]) - done() - }) - }); - - it("it should return data for a valid request", (done) => { - const timestamp = moment().add(config.storage_url_expiry, "seconds").toISOString() - const periodWiseData = { - "2023-06-15": ["file1", "file2"], - }; - chai.spy.on(exhaustService, "getFromStorage", () => { - return Promise.resolve({ - expiresAt: timestamp, - files: ["file1", "file2"], - periodWiseFiles: periodWiseData, - }); - }) - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.VALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.OK) - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - res.body.result.expiresAt.should.be.eq(timestamp) - res.body.result.files.should.have.length(2) - res.body.result.periodWiseFiles.should.be.eql(periodWiseData) - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]) - done() - }) - }); -}) diff --git a/api-service/src/v1/test/Config.ts b/api-service/src/v1/test/Config.ts deleted file mode 100644 index c7a01641..00000000 --- a/api-service/src/v1/test/Config.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { routesConfig } from "../configs/RoutesConfig" -import { config as appConfig } from "../configs/Config"; - -const config = { - apiDruidEndPoint: `${routesConfig.query.native_query.path}`, - apiDruidSqlEndPoint: `${routesConfig.query.sql_query.path}`, - apiDatasetIngestEndPoint: `${routesConfig.data_ingest.path}`, - apiDatasetSaveEndPoint: `${routesConfig.config.dataset.save.path}`, - apiDatasetUpdateEndPoint: `${routesConfig.config.dataset.update.path}`, - apiDatasetReadEndPoint: `${routesConfig.config.dataset.read.path}`, - apiDatasetListEndPoint: `${routesConfig.config.dataset.list.path}`, - apiDatasourceSaveEndPoint: `${routesConfig.config.datasource.save.path}`, - apiDatasourceUpdateEndPoint: `${routesConfig.config.datasource.update.path}`, - apiDatasourceReadEndPoint: `${routesConfig.config.datasource.read.path}`, - apiDatasourceListEndPoint: `${routesConfig.config.datasource.list.path}`, - apiDatasetSourceConfigSaveEndPoint: `${routesConfig.config.dataset_source_config.save.path}`, - apiDatasetSourceConfigUpdateEndPoint: `${routesConfig.config.dataset_source_config.update.path}`, - apiDatasetSourceConfigReadEndPoint: `${routesConfig.config.dataset_source_config.read.path}`, - apiDatasetSourceConfigListEndPoint: `${routesConfig.config.dataset_source_config.list.path}`, - apiExhaustEndPoint: `${routesConfig.exhaust.path}`, - druidHost: `${appConfig.query_api.druid.host}`, - druidPort: `${appConfig.query_api.druid.port}`, - druidEndPoint: `${appConfig.query_api.druid.native_query_path}`, - druidSqlEndPoint: `${appConfig.query_api.druid.sql_query_path}`, - storage_url_expiry: 3600, - exhaustMaxDateRange: 31, - druidDatasourcesEndPoint: `${appConfig.query_api.druid.list_datasources_path}`, - druidSubmitIngestionEndPoint: `/${appConfig.query_api.druid.submit_ingestion}`, - apiSubmitIngestionEndPoint: `${routesConfig.submit_ingestion.path}` -}; -export { config }; diff --git a/api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts b/api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts deleted file mode 100644 index bcf11ac1..00000000 --- a/api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts +++ /dev/null @@ -1,300 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import constants from '../resources/Constants.json' -import { TestDatasetSourceConfig } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { describe, it } from 'mocha'; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { DatasetStatus } from "../models/DatasetModels"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("Dataset source config create API", () => { - it("should insert a record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai.spy.on(ResponseHandler, "successResponse", ()=>{ - throw new Error("Error occured while sending response") - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - chai.spy.restore(ResponseHandler, "successResponse") - done(); - }); - }); - it("should not insert record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record if already exists in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}]) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.CONFLICT); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["409_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record when request object contains missing fields", (done) => { - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.MISSING_REQUIRED_FIELDS_CREATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not insert record when given invalid schema", (done) => { - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) -}) -describe("Dataset source config update API", () => { - it("should successfully update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve() - }) - chai - .request(app) - .patch(config.apiDatasetSourceConfigUpdateEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .patch(config.apiDatasetSourceConfigUpdateEndPoint) - .send(TestDatasetSourceConfig.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.update.api_id) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records when request object does not contain required fields", (done) => { - chai - .request(app) - .patch(config.apiDatasetSourceConfigUpdateEndPoint) - .send(TestDatasetSourceConfig.MISSING_REQUIRED_FIELDS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); -}) -describe("Dataset source config read API", () => { - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([TestDatasetSourceConfig.VALID_RECORD]) - }) - chai - .request(app) - .get(config.apiDatasetSourceConfigReadEndPoint.replace(":datasetId", TestDatasetSourceConfig.SAMPLE_ID).concat(`?status = ${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should throw error if retrieved record is empty", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get(config.apiDatasetSourceConfigReadEndPoint.replace(":datasetId", TestDatasetSourceConfig.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should not retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .get(config.apiDatasetSourceConfigReadEndPoint.replace(":datasetId", TestDatasetSourceConfig.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - -}) -describe("Dataset source config list API", () => { - it("should successfully list records in the table", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}, {}, {}]) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigListEndPoint) - .send(TestDatasetSourceConfig.VALID_LIST_REQUEST_ACTIVE_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("array") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }) - it("should not list records if request object is invalid", (done) => { - chai - .request(app) - .post(config.apiDatasetSourceConfigListEndPoint) - .send({"filters": true}) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not list records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigListEndPoint) - .send(TestDatasetSourceConfig.VALID_LIST_REQUEST_DISABLED_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - -}) diff --git a/api-service/src/v1/test/DatasetTestService.spec.ts b/api-service/src/v1/test/DatasetTestService.spec.ts deleted file mode 100644 index 5b4ab241..00000000 --- a/api-service/src/v1/test/DatasetTestService.spec.ts +++ /dev/null @@ -1,366 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import constants from '../resources/Constants.json' -import { TestDataset } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { Datasets } from "../helpers/Datasets"; -import { describe, it } from 'mocha'; -import { DatasetStatus } from "../models/DatasetModels"; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("Dataset create API", () => { - it("should insert a record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should insert a master dataset record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA_MASTER_DATASET) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record if already exists in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.CONFLICT); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["409_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record when request object contains missing fields", (done) => { - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.MISSING_REQUIRED_FIELDS_CREATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); - it("should not insert record when given invalid schema", (done) => { - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); - it("should not insert the record when there's a duplicate denorm out field", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.DUPLICATE_DENORM_OUT_FIELD) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should insert the record when there's no duplicate denorm out field", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_DENORM_OUT_FIELD) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); -}) -describe("Dataset update API", () => { - beforeEach(() => { - chai.spy.on(dbConnector, "listRecords", () => { - Promise.resolve() - }) - }) - afterEach(() => { - chai.spy.restore(dbConnector, "listRecords") - }) - it("should successfully update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve() - }) - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records when request object does not contain required fields", (done) => { - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.MISSING_REQUIRED_FIELDS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); -}) -describe("Dataset read API", () => { - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([TestDataset.VALID_RECORD]) - }) - chai - .request(app) - .get(config.apiDatasetReadEndPoint.replace(":datasetId", TestDataset.SAMPLE_ID).concat(`?status = ${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should throw error if retrieved record is empty", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get(config.apiDatasetReadEndPoint.replace(":datasetId", TestDataset.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should not retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .get(config.apiDatasetReadEndPoint.replace(":datasetId", TestDataset.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - -}) -describe("Dataset list API", () => { - it("should successfully list records in the table", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}, {}, {}]) - }) - chai - .request(app) - .post(config.apiDatasetListEndPoint) - .send(TestDataset.VALID_LIST_REQUEST_ACTIVE_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("array") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }) - it("should not list records if request object is invalid", (done) => { - chai - .request(app) - .post(config.apiDatasetListEndPoint) - .send({ - "filters": true - }) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not list records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetListEndPoint) - .send(TestDataset.VALID_LIST_REQUEST_DISABLED_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) -}) - -describe("Error scenarios in Dataset API", () => { - it("should not update records in database", (done) => { - chai.spy.on(Datasets.prototype, "getValues", () => { - throw new Error("error occured while parsing data") - }) - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(Datasets.prototype, "getValues") - done() - }) - }) -}) diff --git a/api-service/src/v1/test/DatasourceTestService.spec.ts b/api-service/src/v1/test/DatasourceTestService.spec.ts deleted file mode 100644 index e6df8718..00000000 --- a/api-service/src/v1/test/DatasourceTestService.spec.ts +++ /dev/null @@ -1,493 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import constants from '../resources/Constants.json' -import { TestDataSource } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { describe, it } from 'mocha'; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { ingestorService } from "../routes/Router"; -import { DatasetStatus } from "../models/DatasetModels"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe('Datasource APIS', () => { - afterEach(()=>{ - chai.spy.restore() - }) - describe("Datasource create API", () => { - it("should insert a record in the database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_SAVED) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error when datasource ref is invalid", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.INVALID_DATASOURCE_REF) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error when topic is invalid", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.INVALID_INPUT_TOPIC) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error when dataset id does not exists", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return {} - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "404_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert a record when it already exists in database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ {} ]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - - res.should.have.status(httpStatus.CONFLICT); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "409_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai.spy.on(ResponseHandler, "successResponse", () => { - throw new Error("Error occured while sending response") - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - chai.spy.restore(ResponseHandler, "successResponse") - done(); - }); - }); - it("should not insert record in the database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occured while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record when request object contains missing fields", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.MISSING_REQUIRED_FIELDS_CREATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - done(); - }); - }) - it("should not insert record when given invalid schema", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - done(); - }); - }) - it("should create datalake datasource", (done)=>{ - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_DATALAKE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_SAVED) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }) - }) - describe("Datasource update API", () => { - it("should not update records in database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occured while connecting to postgres")) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should successfully update records in database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_UPDATED) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - - it("should not update records when request object does not contain required fields", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.MISSING_REQUIRED_FIELDS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - done(); - }); - }); - it("should update datalake datasource", (done)=>{ - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.VALID_DATALAKE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_UPDATED) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }) - it("should not update records when given invalid schema in case of datalake", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.INVALID_DATALAKE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }) - }) - describe("Datasource read API", () => { - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ TestDataSource.VALID_RECORD ]) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID).concat(`?status=${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ TestDataSource.VALID_RECORD ]) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should throw error if records are empty", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID).concat(`?status=${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "404_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should not retrieve records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID).concat(`?status=${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - - }) - describe("Datasource list API", () => { - it("should successfully list records in the table", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ {} ]) - }) - chai - .request(app) - .post(config.apiDatasourceListEndPoint) - .send(TestDataSource.VALID_LIST_REQUEST_DISABLED_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("array") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }) - it("should not list records if request object is invalid", (done) => { - chai - .request(app) - .post(config.apiDatasourceListEndPoint) - .send({ "filters": true }) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not list records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasourceListEndPoint) - .send(TestDataSource.VALID_LIST_REQUEST_ACTIVE_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - }) -}) - diff --git a/api-service/src/v1/test/Fixtures.ts b/api-service/src/v1/test/Fixtures.ts deleted file mode 100644 index 634db55b..00000000 --- a/api-service/src/v1/test/Fixtures.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { DatasetStatus } from "../models/DatasetModels"; - -class TestDruidQuery { - public static VALID_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"timeseries","dataSource":"telemetry-events","aggregations":[{"type":"count","name":"count"}],"granularity":"all","postAggregations":[],"intervals": "2021-02-19/2021-02-20"}}'; - public static HIGH_DATE_RANGE_GIVEN_AS_LIST = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"groupBy","dataSource":"telemetry-events","dimensions":["actor_type","content_framework"],"limit":15, "metric":"count","granularity":"all","intervals":["2021-01-02/2021-02-05"],"aggregations":[{"type":"count","name":"count"}]}}'; - public static HIGH_DATE_RANGE_GIVEN_AS_STRING = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"topN","dataSource":"telemetry-events","dimension":"actor_id","threshold":10,"metric":"count","granularity":"all","intervals":"2020-12-30/2021-02-02","aggregations":[{"type":"count","name":"count"}]}}'; - public static HIGH_THRESHOLD_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"scan","dataSource":"telemetry-events","dimension":"mid","threshold":1000,"metric":"count","granularity":"all","intervals":["2020-12-31/2021-01-01"],"aggregations":[]}}'; - public static HIGH_LIMIT_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"scan","dataSource":"telemetry-events","granularity":"all","intervals":["2020-12-21/2021-01-01"],"resultFormat":"compactedList","limit":1000,"columns":["actor_id", "mid"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static WITHOUT_THRESOLD_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"timeBoundary","dataSource":"telemetry-events","dimension":"content_status","metric":"count","granularity":"all","intervals":["2020-12-21/2020-12-22"],"aggregations":[]}}'; - public static WITHOUT_DATE_RANGE_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"search","dataSource":"telemetry-events","granularity":"all","intervals":"","resultFormat":"compactedList","columns":["__time"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static UNSUPPORTED_DATA_SOURCE = - '{"context":{"dataSource":"invalid_data_source"},"query":{"queryType":"timeBoundary","dataSource":"invalid_data_source","granularity":"all","intervals":["2022-10-17/2022-10-19"],"resultFormat":"compactedList","columns":["__time","scans"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static INVALID_QUERY_TYPE = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"invalidQueryType", "dataSource":"telemetry-events", "granularity":"all", "intervals":"2021-12-31/2022-01-20"}}'; - public static UNSUPPORTED_SCHEMA = - '{"context":{},"query":{"queryType":"invalidQueryType", "dataSource":"telemetry-events", "granularity":"all", "intervals":"2021-12-31/2022-01-20"}}'; - public static VALID_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT * FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; - public static VALID_QUERY_DATALAKE = - '{"context":{"dataSource":"telemetry-events", "dataSourceType":"datalake"},"querySql":{"query": "SELECT COUNT(*) AS totalRatingsCount FROM \\"d1\\" LIMIT 100"}}'; - public static INVALID_DATASOURCE_TYPE = '{"context":{"dataSource":"telemetry-events", "dataSourceType":"random"},"querySql":{"query": "SELECT COUNT(*) AS totalRatingsCount FROM \\"d1\\" LIMIT 100"}}'; - public static EMPTY_SQL_QUERY = '{"context":{"dataSource":"telemetry-events", "dataSourceType":"datalake"},"querySql":{"query": ""}}'; - public static CASE_INSENSITIVE_SQL_QUERY = '{"context":{"dataSource":"telemetry-events","dataSourceType":"datalake"},"querySql":{"query":"select * from \\"telemetry-events\\" where __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; - public static HIGH_LIMIT_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT mid FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-22\' LIMIT 1000"}}'; - public static WITHOUT_LIMIT_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-02\'"}}'; - public static HIGH_DATE_RANGE_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-02-12\' LIMIT 10"}}'; - public static WITHOUT_DATE_RANGE_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT content_status FROM \\"telemetry-events\\" LIMIT 5"}}'; - public static UNSUPPORTED_DATASOURCE_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT __time FROM \\"invalid-datasource\\" LIMIT 10"}}'; - public static SKIP_VALIDATION_NATIVE = '{"context":{"dataSource":"system-stats"},"query":{"queryType":"timeBoundary","dataSource":"system-stats","granularity":"all","intervals":["2022-10-17/2022-10-19"],"resultFormat":"compactedList","columns":["__time","scans"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static SKIP_VALIDATION_SQL = '{"context":{"dataSource":"failed-events-summary"},"querySql":{"query":"SELECT * FROM \\"failed-events-summary\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; - public static INVALID_SQL_QUERY = '{\"context\":{\"dataSource\":\"system-events\",\"granularity\":\"day\"},\"querySql\":{\"query\":\"SELECT * \"}}'; - public static MISSING_TABLE_NAME = '{\"context\":{\"dataSource\":\"system-events\",\"granularity\":\"day\"},\"querySql\":{\"query\":\"SELECT * FROM \"}}'; -} - -class TestDataIngestion { - public static SAMPLE_INDIVIDUAL_EVENT = { "data": { "event": { "context": { "transaction_id": "3d3bac46-d252-4da0-9290-afdd524d0214", "country": "IND", "bpp_id": "becknify.humbhionline.in.mobility.BPP/beckn_open/app1-succinct-in", "city": "std:080", "message_id": "52dcf5a9-8986-47ff-a9d0-f380b23e3dfe", "core_version": "0.9.1", "ttl": "PT1M", "bap_id": "mobilityreferencebap.becknprotocol.io", "domain": "nic2004:60221", "bpp_uri": "https://becknify.humbhionline.in/mobility/beckn_open/app1-succinct-in/bpp", "action": "on_status", "bap_uri": "https://mobilityreferencebap.becknprotocol.io", "timestamp": "2023-02-22T19:06:27.887Z" }, "message": { "order": { "quote": { "breakup": [ { "price": { "currency": "INR", "value": "58.2936244525222" }, "type": "item", "title": "Fare" }, { "price": { "currency": "INR", "value": "10.492852401453995" }, "type": "item", "title": "Tax" } ], "price": { "currency": "INR", "value": "68.7864768539762" } }, "provider": { "locations": [ { "gps": "12.973437,77.608771", "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.provider_location" } ], "id": "./mobility/ind.blr/7@taxi.becknprotocol.io.provider", "descriptor": { "images": [ "https://taxi.becknprotocol.io/companies/view/7" ], "name": "Best Taxies" }, "categories": [ { "id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "descriptor": { "name": "Premium Taxi" } } ], "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ] }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.order", "state": "Awaiting Driver acceptance", "fulfillment": { "agent": { "phone": "+919082233441", "name": "Michel MJ" }, "start": { "location": { "gps": "12.973437,77.608771" } }, "end": { "location": { "gps": "12.935193,77.624481" } }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "vehicle": { "registration": "KA 05 3456" }, "customer": { "person": { "name": "./Rajat/Mr./Rajat/ /Kumar/" }, "contact": { "phone": "+919867654322", "email": "er.rjtkumar@gmail.com" } } }, "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ], "billing": { "address": { "country": "IND", "door": "MBT", "city": "std:080", "area_code": "560078", "name": "RajatKumar", "locality": "", "building": ",A33" }, "phone": "+919867654322", "name": "./Rajat/Mr./Rajat/ /Kumar/", "email": "er.rjtkumar@gmail.com" } } } } }} - public static SAMPLE_INPUT = { "data": { "id": "beckn-batch-1", "events": [ { "context": { "transaction_id": "3d3bac46-d252-4da0-9290-afdd524d0214", "country": "IND", "bpp_id": "becknify.humbhionline.in.mobility.BPP/beckn_open/app1-succinct-in", "city": "std:080", "message_id": "52dcf5a9-8986-47ff-a9d0-f380b23e3dfe", "core_version": "0.9.1", "ttl": "PT1M", "bap_id": "mobilityreferencebap.becknprotocol.io", "domain": "nic2004:60221", "bpp_uri": "https://becknify.humbhionline.in/mobility/beckn_open/app1-succinct-in/bpp", "action": "on_status", "bap_uri": "https://mobilityreferencebap.becknprotocol.io", "timestamp": "2023-02-22T19:06:27.887Z" }, "message": { "order": { "quote": { "breakup": [ { "price": { "currency": "INR", "value": "58.2936244525222" }, "type": "item", "title": "Fare" }, { "price": { "currency": "INR", "value": "10.492852401453995" }, "type": "item", "title": "Tax" } ], "price": { "currency": "INR", "value": "68.7864768539762" } }, "provider": { "locations": [ { "gps": "12.973437,77.608771", "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.provider_location" } ], "id": "./mobility/ind.blr/7@taxi.becknprotocol.io.provider", "descriptor": { "images": [ "https://taxi.becknprotocol.io/companies/view/7" ], "name": "Best Taxies" }, "categories": [ { "id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "descriptor": { "name": "Premium Taxi" } } ], "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ] }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.order", "state": "Awaiting Driver acceptance", "fulfillment": { "agent": { "phone": "+919082233441", "name": "Michel MJ" }, "start": { "location": { "gps": "12.973437,77.608771" } }, "end": { "location": { "gps": "12.935193,77.624481" } }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "vehicle": { "registration": "KA 05 3456" }, "customer": { "person": { "name": "./Rajat/Mr./Rajat/ /Kumar/" }, "contact": { "phone": "+919867654322", "email": "er.rjtkumar@gmail.com" } } }, "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ], "billing": { "address": { "country": "IND", "door": "MBT", "city": "std:080", "area_code": "560078", "name": "RajatKumar", "locality": "", "building": ",A33" }, "phone": "+919867654322", "name": "./Rajat/Mr./Rajat/ /Kumar/", "email": "er.rjtkumar@gmail.com" } } } }, { "context": { "domain": "nic2004:60221", "country": "IND", "city": "std:080", "core_version": "0.9.1", "action": "track", "bap_id": "mobilityreferencebap.becknprotocol.io", "bap_uri": "https://mobilityreferencebap.becknprotocol.io", "bpp_id": "becknify.humbhionline.in.mobility.BPP/beckn_open/app1-succinct-in", "bpp_uri": "https://becknify.humbhionline.in/mobility/beckn_open/app1-succinct-in/bpp", "transaction_id": "3d3bac46-d252-4da0-9290-afdd524d0214", "message_id": "b52878f3-28ed-4c31-8ebb-8989f33c3220", "timestamp": "2023-02-22T19:07:07.887Z", "ttl": "PT1M" }, "message": { "order_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.order" } } ] } } -} -class TestDataset { - public static VALID_SCHEMA = { "type": "dataset", "dataset_id": "observations", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "tags": [], "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_SCHEMA_MASTER_DATASET = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "tags": [], "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" } - public static VALID_UPDATE_SCHEMA = { "type": "master-dataset", "dataset_id": "observations", "id": "observations", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "router_config": { "topic": "router.topic" }, "tags": [], "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z" } - public static INVALID_SCHEMA = { "dataset_id": "observations", "type": "dataset", "name": "observations", "router_config": { "topic": "" }, "data_schema": "string", "dataset_config": { "entry_topic": "local.ingest", "redis_db_host": "localhost", "redis_db_port": 6379 }, "status": DatasetStatus.Live, "published_date": "2023-03-24 12:19:32.091544" } - public static MISSING_REQUIRED_FIELDS_CREATE = { "type": "dataset", "dataset_id": "observations", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static SAMPLE_ID = "observations"; - public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; - public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; - public static MISSING_REQUIRED_FIELDS_UPDATE = { "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_RECORD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; - public static DUPLICATE_DENORM_OUT_FIELD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }, { "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_DENORM_OUT_FIELD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b91ac", "id": "observctions", "name": "telemetay-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata2" }, { "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; -} - -class TestDataSource { - public static VALID_DATALAKE_SCHEMA = {"id":"sb-telemetry_sb-telemetry","datasource":"sb-telemetry-events","type":"datalake","dataset_id":"sb-telemetry","ingestion_spec":{"dataset":"sb-telemetry-events","schema":{"table":"sb-telemetry-events","partitionColumn":"eid","timestampColumn":"syncts","primaryKey":"mid","columnSpec":[{"type":"string","name":"eid"},{"type":"string","name":"ver"},{"type":"long","name":"syncts"},{"type":"double","name":"ets"},{"type":"boolean","name":"flags_ex_processed"},{"type":"boolean","name":"flags_pp_validation_processed"},{"type":"boolean","name":"flags_pp_duplicate_skipped"},{"type":"boolean","name":"flags_user_denorm"},{"type":"boolean","name":"flags_device_denorm"},{"type":"boolean","name":"flags_loc_denorm"},{"type":"boolean","name":"flags_content_denorm"},{"type":"boolean","name":"flags_coll_denorm"},{"type":"string","name":"mid"},{"type":"string","name":"actor_id"},{"type":"string","name":"actor_type"},{"type":"array","name":"edata_visits"},{"type":"string","name":"edata_type"},{"type":"double","name":"edata_duration"},{"type":"long","name":"edata_size"},{"type":"string","name":"edata_query"},{"type":"array","name":"edata_filters_objectType"},{"type":"array","name":"edata_filters_version"},{"type":"array","name":"edata_filters_status"},{"type":"array","name":"edata_filters_id"},{"type":"boolean","name":"edata_filters_isRootOrg"},{"type":"string","name":"edata_filters_trackable_enabled"},{"type":"json","name":"edata_filters_channel"},{"type":"json","name":"edata_filters_framework"},{"type":"json","name":"edata_filters_resourceType"},{"type":"array","name":"edata_filters_identifier"},{"type":"array","name":"edata_filters_contentType"},{"type":"array","name":"edata_filters_mimeType"},{"type":"array","name":"edata_filters_hashTagId"},{"type":"long","name":"edata_filters_compatibilityLevel_min"},{"type":"long","name":"edata_filters_compatibilityLevel_max"},{"type":"string","name":"edata_filters_createdBy"},{"type":"array","name":"edata_filters_mediaType"},{"type":"string","name":"edata_filters_origin"},{"type":"array","name":"edata_filters_primaryCategory"},{"name":"edata_filters_trackable"},{"type":"string","name":"edata_sort_lastUpdatedOn"},{"type":"array","name":"edata_topn"},{"type":"string","name":"edata_pageid"},{"type":"string","name":"edata_uri"},{"type":"string","name":"edata_subtype"},{"type":"string","name":"edata_id"},{"type":"string","name":"edata_data"},{"type":"string","name":"edata_uaspec_agent"},{"type":"string","name":"edata_uaspec_ver"},{"type":"string","name":"edata_uaspec_system"},{"type":"string","name":"edata_uaspec_platform"},{"type":"string","name":"edata_uaspec_raw"},{"type":"string","name":"edata_state"},{"type":"array","name":"edata_props"},{"type":"string","name":"edata_prevstate"},{"type":"string","name":"edata_dspec_os"},{"type":"string","name":"edata_dspec_make"},{"type":"string","name":"edata_dspec_id"},{"type":"double","name":"edata_dspec_idisk"},{"type":"double","name":"edata_dspec_edisk"},{"type":"double","name":"edata_dspec_scrn"},{"type":"string","name":"edata_dspec_camera"},{"type":"string","name":"edata_dspec_cpu"},{"type":"long","name":"edata_dspec_sims"},{"type":"string","name":"edata_dspec_webview"},{"type":"array","name":"edata_extra_pos"},{"type":"array","name":"edata_extra_values"},{"type":"string","name":"edata_extra_query"},{"type":"string","name":"edata_mode"},{"type":"timestamp","name":"@timestamp"},{"type":"string","name":"context_channel"},{"type":"string","name":"context_pdata_id"},{"type":"string","name":"context_pdata_ver"},{"type":"string","name":"context_pdata_pid"},{"type":"string","name":"context_env"},{"type":"string","name":"context_sid"},{"type":"string","name":"context_rollup_l1"},{"type":"string","name":"context_rollup_l2"},{"type":"string","name":"context_rollup_l3"},{"type":"array","name":"context_cdata"},{"type":"string","name":"context_did"},{"type":"string","name":"context_uid"},{"type":"string","name":"object_id"},{"type":"string","name":"object_type"},{"type":"string","name":"object_ver"},{"type":"string","name":"object_rollup_l1"},{"type":"string","name":"object_version"},{"type":"array","name":"tags"},{"type":"string","name":"obsrv_meta_source_connector"},{"type":"string","name":"obsrv_meta_source_id"}]},"inputFormat":{"type":"json","flattenSpec":{"fields":[{"type":"path","expr":"$.eid","name":"eid"},{"type":"path","expr":"$.ver","name":"ver"},{"type":"path","expr":"$.syncts","name":"syncts"},{"type":"path","expr":"$.ets","name":"ets"},{"type":"path","expr":"$.flags.ex_processed","name":"flags_ex_processed"},{"type":"path","expr":"$.flags.pp_validation_processed","name":"flags_pp_validation_processed"},{"type":"path","expr":"$.flags.pp_duplicate_skipped","name":"flags_pp_duplicate_skipped"},{"type":"path","expr":"$.flags.user_denorm","name":"flags_user_denorm"},{"type":"path","expr":"$.flags.device_denorm","name":"flags_device_denorm"},{"type":"path","expr":"$.flags.loc_denorm","name":"flags_loc_denorm"},{"type":"path","expr":"$.flags.content_denorm","name":"flags_content_denorm"},{"type":"path","expr":"$.flags.coll_denorm","name":"flags_coll_denorm"},{"type":"path","expr":"$.mid","name":"mid"},{"type":"path","expr":"$.actor.id","name":"actor_id"},{"type":"path","expr":"$.actor.type","name":"actor_type"},{"type":"path","expr":"$.edata.visits[*]","name":"edata_visits"},{"type":"path","expr":"$.edata.type","name":"edata_type"},{"type":"path","expr":"$.edata.duration","name":"edata_duration"},{"type":"path","expr":"$.edata.size","name":"edata_size"},{"type":"path","expr":"$.edata.query","name":"edata_query"},{"type":"path","expr":"$.edata.filters.objectType[*]","name":"edata_filters_objectType"},{"type":"path","expr":"$.edata.filters.version[*]","name":"edata_filters_version"},{"type":"path","expr":"$.edata.filters.status[*]","name":"edata_filters_status"},{"type":"path","expr":"$.edata.filters.id[*]","name":"edata_filters_id"},{"type":"path","expr":"$.edata.filters.isRootOrg","name":"edata_filters_isRootOrg"},{"type":"path","expr":"$.edata.filters.trackable.enabled","name":"edata_filters_trackable_enabled"},{"type":"path","expr":"$.edata.filters.channel","name":"edata_filters_channel"},{"type":"path","expr":"$.edata.filters.framework","name":"edata_filters_framework"},{"type":"path","expr":"$.edata.filters.resourceType","name":"edata_filters_resourceType"},{"type":"path","expr":"$.edata.filters.identifier[*]","name":"edata_filters_identifier"},{"type":"path","expr":"$.edata.filters.contentType[*]","name":"edata_filters_contentType"},{"type":"path","expr":"$.edata.filters.mimeType[*]","name":"edata_filters_mimeType"},{"type":"path","expr":"$.edata.filters.hashTagId[*]","name":"edata_filters_hashTagId"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.min","name":"edata_filters_compatibilityLevel_min"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.max","name":"edata_filters_compatibilityLevel_max"},{"type":"path","expr":"$.edata.filters.createdBy","name":"edata_filters_createdBy"},{"type":"path","expr":"$.edata.filters.mediaType[*]","name":"edata_filters_mediaType"},{"type":"path","expr":"$.edata.filters.origin","name":"edata_filters_origin"},{"type":"path","expr":"$.edata.filters.primaryCategory[*]","name":"edata_filters_primaryCategory"},{"type":"path","expr":"$.edata.filters.trackable","name":"edata_filters_trackable"},{"type":"path","expr":"$.edata.sort.lastUpdatedOn","name":"edata_sort_lastUpdatedOn"},{"type":"path","expr":"$.edata.topn[*]","name":"edata_topn"},{"type":"path","expr":"$.edata.pageid","name":"edata_pageid"},{"type":"path","expr":"$.edata.uri","name":"edata_uri"},{"type":"path","expr":"$.edata.subtype","name":"edata_subtype"},{"type":"path","expr":"$.edata.id","name":"edata_id"},{"type":"path","expr":"$.edata.data","name":"edata_data"},{"type":"path","expr":"$.edata.uaspec.agent","name":"edata_uaspec_agent"},{"type":"path","expr":"$.edata.uaspec.ver","name":"edata_uaspec_ver"},{"type":"path","expr":"$.edata.uaspec.system","name":"edata_uaspec_system"},{"type":"path","expr":"$.edata.uaspec.platform","name":"edata_uaspec_platform"},{"type":"path","expr":"$.edata.uaspec.raw","name":"edata_uaspec_raw"},{"type":"path","expr":"$.edata.state","name":"edata_state"},{"type":"path","expr":"$.edata.props[*]","name":"edata_props"},{"type":"path","expr":"$.edata.prevstate","name":"edata_prevstate"},{"type":"path","expr":"$.edata.dspec.os","name":"edata_dspec_os"},{"type":"path","expr":"$.edata.dspec.make","name":"edata_dspec_make"},{"type":"path","expr":"$.edata.dspec.id","name":"edata_dspec_id"},{"type":"path","expr":"$.edata.dspec.idisk","name":"edata_dspec_idisk"},{"type":"path","expr":"$.edata.dspec.edisk","name":"edata_dspec_edisk"},{"type":"path","expr":"$.edata.dspec.scrn","name":"edata_dspec_scrn"},{"type":"path","expr":"$.edata.dspec.camera","name":"edata_dspec_camera"},{"type":"path","expr":"$.edata.dspec.cpu","name":"edata_dspec_cpu"},{"type":"path","expr":"$.edata.dspec.sims","name":"edata_dspec_sims"},{"type":"path","expr":"$.edata.dspec.webview","name":"edata_dspec_webview"},{"type":"path","expr":"$.edata.extra.pos[*]","name":"edata_extra_pos"},{"type":"path","expr":"$.edata.extra.values[*]","name":"edata_extra_values"},{"type":"path","expr":"$.edata.extra.query","name":"edata_extra_query"},{"type":"path","expr":"$.edata.mode","name":"edata_mode"},{"type":"path","expr":"$.@timestamp","name":"@timestamp"},{"type":"path","expr":"$.context.channel","name":"context_channel"},{"type":"path","expr":"$.context.pdata.id","name":"context_pdata_id"},{"type":"path","expr":"$.context.pdata.ver","name":"context_pdata_ver"},{"type":"path","expr":"$.context.pdata.pid","name":"context_pdata_pid"},{"type":"path","expr":"$.context.env","name":"context_env"},{"type":"path","expr":"$.context.sid","name":"context_sid"},{"type":"path","expr":"$.context.rollup.l1","name":"context_rollup_l1"},{"type":"path","expr":"$.context.rollup.l2","name":"context_rollup_l2"},{"type":"path","expr":"$.context.rollup.l3","name":"context_rollup_l3"},{"type":"path","expr":"$.context.cdata[*]","name":"context_cdata"},{"type":"path","expr":"$.context.did","name":"context_did"},{"type":"path","expr":"$.context.uid","name":"context_uid"},{"type":"path","expr":"$.object.id","name":"object_id"},{"type":"path","expr":"$.object.type","name":"object_type"},{"type":"path","expr":"$.object.ver","name":"object_ver"},{"type":"path","expr":"$.object.rollup.l1","name":"object_rollup_l1"},{"type":"path","expr":"$.object.version","name":"object_version"},{"type":"path","expr":"$.tags[*]","name":"tags"},{"type":"path","expr":"$.obsrv_meta.source.connector","name":"obsrv_meta_source_connector"},{"type":"path","expr":"$.obsrv_meta.source.connectorInstance","name":"obsrv_meta_source_id"}]}}},"datasource_ref":"sb-telemetry-events","retention_period":{"enabled":"false"},"archival_policy":{"enabled":"false"},"purge_policy":{"enabled":"false"},"backup_config":{"enabled":"false"},"status":"Live","created_by":"SYSTEM","updated_by":"SYSTEM","published_date":"2023-07-03 00:00:00"} - public static INVALID_DATALAKE_SCHEMA = {"id":"sb-telemetry_sb-telemetry","datasource":"sb-telemetry-events","type":"invalid_type","dataset_id":"sb-telemetry","ingestion_spec":{"dataset":"sb-telemetry-events","schema":{"table":"table_name_doesnt_match_ref","partitionColumn":"eid","timestampColumn":"syncts","primaryKey":"mid","columnSpec":[{"type":"string","name":"eid"},{"type":"string","name":"ver"},{"type":"long","name":"syncts"},{"type":"double","name":"ets"},{"type":"boolean","name":"flags_ex_processed"},{"type":"boolean","name":"flags_pp_validation_processed"},{"type":"boolean","name":"flags_pp_duplicate_skipped"},{"type":"boolean","name":"flags_user_denorm"},{"type":"boolean","name":"flags_device_denorm"},{"type":"boolean","name":"flags_loc_denorm"},{"type":"boolean","name":"flags_content_denorm"},{"type":"boolean","name":"flags_coll_denorm"},{"type":"string","name":"mid"},{"type":"string","name":"actor_id"},{"type":"string","name":"actor_type"},{"type":"array","name":"edata_visits"},{"type":"string","name":"edata_type"},{"type":"double","name":"edata_duration"},{"type":"long","name":"edata_size"},{"type":"string","name":"edata_query"},{"type":"array","name":"edata_filters_objectType"},{"type":"array","name":"edata_filters_version"},{"type":"array","name":"edata_filters_status"},{"type":"array","name":"edata_filters_id"},{"type":"boolean","name":"edata_filters_isRootOrg"},{"type":"string","name":"edata_filters_trackable_enabled"},{"type":"json","name":"edata_filters_channel"},{"type":"json","name":"edata_filters_framework"},{"type":"json","name":"edata_filters_resourceType"},{"type":"array","name":"edata_filters_identifier"},{"type":"array","name":"edata_filters_contentType"},{"type":"array","name":"edata_filters_mimeType"},{"type":"array","name":"edata_filters_hashTagId"},{"type":"long","name":"edata_filters_compatibilityLevel_min"},{"type":"long","name":"edata_filters_compatibilityLevel_max"},{"type":"string","name":"edata_filters_createdBy"},{"type":"array","name":"edata_filters_mediaType"},{"type":"string","name":"edata_filters_origin"},{"type":"array","name":"edata_filters_primaryCategory"},{"name":"edata_filters_trackable"},{"type":"string","name":"edata_sort_lastUpdatedOn"},{"type":"array","name":"edata_topn"},{"type":"string","name":"edata_pageid"},{"type":"string","name":"edata_uri"},{"type":"string","name":"edata_subtype"},{"type":"string","name":"edata_id"},{"type":"string","name":"edata_data"},{"type":"string","name":"edata_uaspec_agent"},{"type":"string","name":"edata_uaspec_ver"},{"type":"string","name":"edata_uaspec_system"},{"type":"string","name":"edata_uaspec_platform"},{"type":"string","name":"edata_uaspec_raw"},{"type":"string","name":"edata_state"},{"type":"array","name":"edata_props"},{"type":"string","name":"edata_prevstate"},{"type":"string","name":"edata_dspec_os"},{"type":"string","name":"edata_dspec_make"},{"type":"string","name":"edata_dspec_id"},{"type":"double","name":"edata_dspec_idisk"},{"type":"double","name":"edata_dspec_edisk"},{"type":"double","name":"edata_dspec_scrn"},{"type":"string","name":"edata_dspec_camera"},{"type":"string","name":"edata_dspec_cpu"},{"type":"long","name":"edata_dspec_sims"},{"type":"string","name":"edata_dspec_webview"},{"type":"array","name":"edata_extra_pos"},{"type":"array","name":"edata_extra_values"},{"type":"string","name":"edata_extra_query"},{"type":"string","name":"edata_mode"},{"type":"timestamp","name":"@timestamp"},{"type":"string","name":"context_channel"},{"type":"string","name":"context_pdata_id"},{"type":"string","name":"context_pdata_ver"},{"type":"string","name":"context_pdata_pid"},{"type":"string","name":"context_env"},{"type":"string","name":"context_sid"},{"type":"string","name":"context_rollup_l1"},{"type":"string","name":"context_rollup_l2"},{"type":"string","name":"context_rollup_l3"},{"type":"array","name":"context_cdata"},{"type":"string","name":"context_did"},{"type":"string","name":"context_uid"},{"type":"string","name":"object_id"},{"type":"string","name":"object_type"},{"type":"string","name":"object_ver"},{"type":"string","name":"object_rollup_l1"},{"type":"string","name":"object_version"},{"type":"array","name":"tags"},{"type":"string","name":"obsrv_meta_source_connector"},{"type":"string","name":"obsrv_meta_source_id"}]},"inputFormat":{"type":"json","flattenSpec":{"fields":[{"type":"path","expr":"$.eid","name":"eid"},{"type":"path","expr":"$.ver","name":"ver"},{"type":"path","expr":"$.syncts","name":"syncts"},{"type":"path","expr":"$.ets","name":"ets"},{"type":"path","expr":"$.flags.ex_processed","name":"flags_ex_processed"},{"type":"path","expr":"$.flags.pp_validation_processed","name":"flags_pp_validation_processed"},{"type":"path","expr":"$.flags.pp_duplicate_skipped","name":"flags_pp_duplicate_skipped"},{"type":"path","expr":"$.flags.user_denorm","name":"flags_user_denorm"},{"type":"path","expr":"$.flags.device_denorm","name":"flags_device_denorm"},{"type":"path","expr":"$.flags.loc_denorm","name":"flags_loc_denorm"},{"type":"path","expr":"$.flags.content_denorm","name":"flags_content_denorm"},{"type":"path","expr":"$.flags.coll_denorm","name":"flags_coll_denorm"},{"type":"path","expr":"$.mid","name":"mid"},{"type":"path","expr":"$.actor.id","name":"actor_id"},{"type":"path","expr":"$.actor.type","name":"actor_type"},{"type":"path","expr":"$.edata.visits[*]","name":"edata_visits"},{"type":"path","expr":"$.edata.type","name":"edata_type"},{"type":"path","expr":"$.edata.duration","name":"edata_duration"},{"type":"path","expr":"$.edata.size","name":"edata_size"},{"type":"path","expr":"$.edata.query","name":"edata_query"},{"type":"path","expr":"$.edata.filters.objectType[*]","name":"edata_filters_objectType"},{"type":"path","expr":"$.edata.filters.version[*]","name":"edata_filters_version"},{"type":"path","expr":"$.edata.filters.status[*]","name":"edata_filters_status"},{"type":"path","expr":"$.edata.filters.id[*]","name":"edata_filters_id"},{"type":"path","expr":"$.edata.filters.isRootOrg","name":"edata_filters_isRootOrg"},{"type":"path","expr":"$.edata.filters.trackable.enabled","name":"edata_filters_trackable_enabled"},{"type":"path","expr":"$.edata.filters.channel","name":"edata_filters_channel"},{"type":"path","expr":"$.edata.filters.framework","name":"edata_filters_framework"},{"type":"path","expr":"$.edata.filters.resourceType","name":"edata_filters_resourceType"},{"type":"path","expr":"$.edata.filters.identifier[*]","name":"edata_filters_identifier"},{"type":"path","expr":"$.edata.filters.contentType[*]","name":"edata_filters_contentType"},{"type":"path","expr":"$.edata.filters.mimeType[*]","name":"edata_filters_mimeType"},{"type":"path","expr":"$.edata.filters.hashTagId[*]","name":"edata_filters_hashTagId"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.min","name":"edata_filters_compatibilityLevel_min"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.max","name":"edata_filters_compatibilityLevel_max"},{"type":"path","expr":"$.edata.filters.createdBy","name":"edata_filters_createdBy"},{"type":"path","expr":"$.edata.filters.mediaType[*]","name":"edata_filters_mediaType"},{"type":"path","expr":"$.edata.filters.origin","name":"edata_filters_origin"},{"type":"path","expr":"$.edata.filters.primaryCategory[*]","name":"edata_filters_primaryCategory"},{"type":"path","expr":"$.edata.filters.trackable","name":"edata_filters_trackable"},{"type":"path","expr":"$.edata.sort.lastUpdatedOn","name":"edata_sort_lastUpdatedOn"},{"type":"path","expr":"$.edata.topn[*]","name":"edata_topn"},{"type":"path","expr":"$.edata.pageid","name":"edata_pageid"},{"type":"path","expr":"$.edata.uri","name":"edata_uri"},{"type":"path","expr":"$.edata.subtype","name":"edata_subtype"},{"type":"path","expr":"$.edata.id","name":"edata_id"},{"type":"path","expr":"$.edata.data","name":"edata_data"},{"type":"path","expr":"$.edata.uaspec.agent","name":"edata_uaspec_agent"},{"type":"path","expr":"$.edata.uaspec.ver","name":"edata_uaspec_ver"},{"type":"path","expr":"$.edata.uaspec.system","name":"edata_uaspec_system"},{"type":"path","expr":"$.edata.uaspec.platform","name":"edata_uaspec_platform"},{"type":"path","expr":"$.edata.uaspec.raw","name":"edata_uaspec_raw"},{"type":"path","expr":"$.edata.state","name":"edata_state"},{"type":"path","expr":"$.edata.props[*]","name":"edata_props"},{"type":"path","expr":"$.edata.prevstate","name":"edata_prevstate"},{"type":"path","expr":"$.edata.dspec.os","name":"edata_dspec_os"},{"type":"path","expr":"$.edata.dspec.make","name":"edata_dspec_make"},{"type":"path","expr":"$.edata.dspec.id","name":"edata_dspec_id"},{"type":"path","expr":"$.edata.dspec.idisk","name":"edata_dspec_idisk"},{"type":"path","expr":"$.edata.dspec.edisk","name":"edata_dspec_edisk"},{"type":"path","expr":"$.edata.dspec.scrn","name":"edata_dspec_scrn"},{"type":"path","expr":"$.edata.dspec.camera","name":"edata_dspec_camera"},{"type":"path","expr":"$.edata.dspec.cpu","name":"edata_dspec_cpu"},{"type":"path","expr":"$.edata.dspec.sims","name":"edata_dspec_sims"},{"type":"path","expr":"$.edata.dspec.webview","name":"edata_dspec_webview"},{"type":"path","expr":"$.edata.extra.pos[*]","name":"edata_extra_pos"},{"type":"path","expr":"$.edata.extra.values[*]","name":"edata_extra_values"},{"type":"path","expr":"$.edata.extra.query","name":"edata_extra_query"},{"type":"path","expr":"$.edata.mode","name":"edata_mode"},{"type":"path","expr":"$.@timestamp","name":"@timestamp"},{"type":"path","expr":"$.context.channel","name":"context_channel"},{"type":"path","expr":"$.context.pdata.id","name":"context_pdata_id"},{"type":"path","expr":"$.context.pdata.ver","name":"context_pdata_ver"},{"type":"path","expr":"$.context.pdata.pid","name":"context_pdata_pid"},{"type":"path","expr":"$.context.env","name":"context_env"},{"type":"path","expr":"$.context.sid","name":"context_sid"},{"type":"path","expr":"$.context.rollup.l1","name":"context_rollup_l1"},{"type":"path","expr":"$.context.rollup.l2","name":"context_rollup_l2"},{"type":"path","expr":"$.context.rollup.l3","name":"context_rollup_l3"},{"type":"path","expr":"$.context.cdata[*]","name":"context_cdata"},{"type":"path","expr":"$.context.did","name":"context_did"},{"type":"path","expr":"$.context.uid","name":"context_uid"},{"type":"path","expr":"$.object.id","name":"object_id"},{"type":"path","expr":"$.object.type","name":"object_type"},{"type":"path","expr":"$.object.ver","name":"object_ver"},{"type":"path","expr":"$.object.rollup.l1","name":"object_rollup_l1"},{"type":"path","expr":"$.object.version","name":"object_version"},{"type":"path","expr":"$.tags[*]","name":"tags"},{"type":"path","expr":"$.obsrv_meta.source.connector","name":"obsrv_meta_source_connector"},{"type":"path","expr":"$.obsrv_meta.source.connectorInstance","name":"obsrv_meta_source_id"}]}}},"datasource_ref":"sb-telemetry-events","retention_period":{"enabled":"false"},"archival_policy":{"enabled":"false"},"purge_policy":{"enabled":"false"},"backup_config":{"enabled":"false"},"status":"Live","created_by":"SYSTEM","updated_by":"SYSTEM","published_date":"2023-07-03 00:00:00"} - public static VALID_SCHEMA = { "dataset_id": "telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "telemetry-events", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "fromDate" }, { "type": "string", "name": "toDate" }, { "type": "string", "name": "tes" }, { "type": "string", "name": "uid" }, { "type": "string", "name": "mobile" }, { "type": "string", "name": "ip" }, { "type": "string", "name": "ipv6" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "string", "name": "derivedlocationdata_district" }, { "type": "string", "name": "derivedlocationdata_from" }, { "type": "string", "name": "derivedlocationdata_state" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "type" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "edata_type" }, { "type": "string", "name": "edata_query" }, { "type": "string", "name": "edata_filters_slug" }, { "type": "boolean", "name": "edata_filters_isTenant" }, { "type": "string", "name": "edata_filters_make_type" }, { "type": "string", "name": "edata_topn[*]_id" }, { "name": "edata_items" }, { "type": "array", "name": "userdata_subject" }, { "type": "string", "name": "userdata_district" }, { "type": "string", "name": "userdata_usersubtype" }, { "type": "array", "name": "userdata_grade" }, { "type": "string", "name": "userdata_usersignintype" }, { "type": "string", "name": "userdata_usertype" }, { "type": "string", "name": "userdata_userlogintype" }, { "type": "string", "name": "userdata_state" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "devicedata_statecustomcode" }, { "type": "string", "name": "devicedata_country" }, { "type": "string", "name": "devicedata_iso3166statecode" }, { "type": "string", "name": "devicedata_city" }, { "type": "string", "name": "devicedata_countrycode" }, { "type": "string", "name": "devicedata_state" }, { "type": "string", "name": "devicedata_devicespec_idisk" }, { "type": "string", "name": "devicedata_devicespec_webview" }, { "type": "string", "name": "devicedata_devicespec_os" }, { "type": "string", "name": "devicedata_devicespec_scrn" }, { "type": "string", "name": "devicedata_devicespec_sims" }, { "type": "string", "name": "devicedata_devicespec_cpu" }, { "type": "string", "name": "devicedata_devicespec_id" }, { "type": "string", "name": "devicedata_devicespec_camera" }, { "type": "string", "name": "devicedata_devicespec_edisk" }, { "type": "string", "name": "devicedata_devicespec_make" }, { "type": "string", "name": "devicedata_statecode" }, { "type": "string", "name": "devicedata_districtcustom" }, { "type": "string", "name": "devicedata_statecustomname" }, { "type": "string", "name": "devicedata_userdeclared_district" }, { "type": "string", "name": "devicedata_userdeclared_state" }, { "type": "string", "name": "context_cdata[*]_id" }, { "type": "string", "name": "context_cdata[*]_type" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_version" }, { "type": "string", "name": "ver" } ] }, "timestampSpec": { "column": "arrival-time", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "eid", "fieldName": "eid" }, { "type": "doubleSum", "name": "syncts", "fieldName": "syncts" }, { "type": "doubleSum", "name": "ets", "fieldName": "ets" }, { "type": "doubleSum", "name": "edata_duration", "fieldName": "edata_duration" }, { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_make_vers", "fieldName": "edata_filters_make_vers" }, { "type": "doubleSum", "name": "devicedata_firstaccess", "fieldName": "devicedata_firstaccess" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxRowsPerSegment": 50000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "telemetry", "consumerProperties": {}, "taskCount": 1, "replicas": 1, "taskDuration": "PT8H", "useEarliestOffset": false, "completionTimeout": "PT8H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.make.vers", "name": "edata_filters_make_vers" }, { "type": "path", "expr": "$.devicedata.firstaccess", "name": "devicedata_firstaccess" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.derivedlocationdata.district", "name": "derivedlocationdata_district" }, { "type": "path", "expr": "$.derivedlocationdata.from", "name": "derivedlocationdata_from" }, { "type": "path", "expr": "$.derivedlocationdata.state", "name": "derivedlocationdata_state" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.type", "name": "type" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.slug", "name": "edata_filters_slug" }, { "type": "path", "expr": "$.edata.filters.isTenant", "name": "edata_filters_isTenant" }, { "type": "path", "expr": "$.edata.filters.make.type", "name": "edata_filters_make_type" }, { "type": "path", "expr": "$.edata.topn[*].id", "name": "edata_topn[*]_id" }, { "type": "path", "expr": "$.edata.items", "name": "edata_items" }, { "type": "path", "expr": "$.userdata.subject[*]", "name": "userdata_subject" }, { "type": "path", "expr": "$.userdata.district", "name": "userdata_district" }, { "type": "path", "expr": "$.userdata.usersubtype", "name": "userdata_usersubtype" }, { "type": "path", "expr": "$.userdata.grade[*]", "name": "userdata_grade" }, { "type": "path", "expr": "$.userdata.usersignintype", "name": "userdata_usersignintype" }, { "type": "path", "expr": "$.userdata.usertype", "name": "userdata_usertype" }, { "type": "path", "expr": "$.userdata.userlogintype", "name": "userdata_userlogintype" }, { "type": "path", "expr": "$.userdata.state", "name": "userdata_state" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.devicedata.statecustomcode", "name": "devicedata_statecustomcode" }, { "type": "path", "expr": "$.devicedata.country", "name": "devicedata_country" }, { "type": "path", "expr": "$.devicedata.iso3166statecode", "name": "devicedata_iso3166statecode" }, { "type": "path", "expr": "$.devicedata.city", "name": "devicedata_city" }, { "type": "path", "expr": "$.devicedata.countrycode", "name": "devicedata_countrycode" }, { "type": "path", "expr": "$.devicedata.state", "name": "devicedata_state" }, { "type": "path", "expr": "$.devicedata.devicespec.idisk", "name": "devicedata_devicespec_idisk" }, { "type": "path", "expr": "$.devicedata.devicespec.webview", "name": "devicedata_devicespec_webview" }, { "type": "path", "expr": "$.devicedata.devicespec.os", "name": "devicedata_devicespec_os" }, { "type": "path", "expr": "$.devicedata.devicespec.scrn", "name": "devicedata_devicespec_scrn" }, { "type": "path", "expr": "$.devicedata.devicespec.sims", "name": "devicedata_devicespec_sims" }, { "type": "path", "expr": "$.devicedata.devicespec.cpu", "name": "devicedata_devicespec_cpu" }, { "type": "path", "expr": "$.devicedata.devicespec.id", "name": "devicedata_devicespec_id" }, { "type": "path", "expr": "$.devicedata.devicespec.camera", "name": "devicedata_devicespec_camera" }, { "type": "path", "expr": "$.devicedata.devicespec.edisk", "name": "devicedata_devicespec_edisk" }, { "type": "path", "expr": "$.devicedata.devicespec.make", "name": "devicedata_devicespec_make" }, { "type": "path", "expr": "$.devicedata.statecode", "name": "devicedata_statecode" }, { "type": "path", "expr": "$.devicedata.districtcustom", "name": "devicedata_districtcustom" }, { "type": "path", "expr": "$.devicedata.statecustomname", "name": "devicedata_statecustomname" }, { "type": "path", "expr": "$.devicedata.userdeclared.district", "name": "devicedata_userdeclared_district" }, { "type": "path", "expr": "$.devicedata.userdeclared.state", "name": "devicedata_userdeclared_state" }, { "type": "path", "expr": "$.context.cdata[*].id", "name": "context_cdata[*]_id" }, { "type": "path", "expr": "$.context.cdata[*].type", "name": "context_cdata[*]_type" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.ver", "name": "ver" } ] } }, "appendToExisting": false } } }, "type":"druid", "datasource": "telemetry-events", "datasource_ref": "telemetry-events", "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_UPDATE_SCHEMA = { "id": "telemetry_telemetry-events", "dataset_id": "telemetry", "type":"druid", "datasource": "telemetry-events", "backup_config": { "enabled": true }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z", "datasource_ref": "telemetry-events" }; - public static INVALID_SCHEMA = { "dataset_id": "telemetry", "ingestion_spec": "invalid data type", "type":"druid", "datasource": "telemetry-events", "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z", "datasource_ref": "telemetry-events", }; - public static INVALID_INPUT_TOPIC = { "id": "sb-telemetry_sb-telemetry", "type":"druid", "datasource": "sb-telemetry", "dataset_id": "sb-telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "sb-telemetry.1_DAY", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "eid" }, { "type": "string", "name": "ver" }, { "type": "long", "name": "syncts" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "boolean", "name": "flags_content_denorm" }, { "type": "boolean", "name": "flags_coll_denorm" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "edata_type" }, { "type": "long", "name": "edata_duration" }, { "type": "string", "name": "edata_query" }, { "type": "array", "name": "edata_filters_objectType" }, { "type": "array", "name": "edata_filters_version" }, { "type": "array", "name": "edata_filters_status" }, { "type": "array", "name": "edata_filters_id" }, { "type": "boolean", "name": "edata_filters_isRootOrg" }, { "type": "string", "name": "edata_filters_trackable_enabled" }, { "type": "array", "name": "edata_filters_identifier" }, { "type": "array", "name": "edata_filters_contentType" }, { "type": "array", "name": "edata_filters_mimeType" }, { "type": "array", "name": "edata_filters_hashTagId" }, { "type": "string", "name": "edata_filters_createdBy" }, { "type": "array", "name": "edata_filters_mediaType" }, { "type": "string", "name": "edata_filters_origin" }, { "type": "array", "name": "edata_filters_primaryCategory" }, { "name": "edata_filters_trackable" }, { "type": "string", "name": "edata_sort_lastUpdatedOn" }, { "type": "array", "name": "edata_topn" }, { "type": "string", "name": "edata_pageid" }, { "type": "string", "name": "edata_uri" }, { "type": "string", "name": "edata_subtype" }, { "type": "string", "name": "edata_id" }, { "type": "string", "name": "edata_data" }, { "type": "string", "name": "edata_uaspec_agent" }, { "type": "string", "name": "edata_uaspec_ver" }, { "type": "string", "name": "edata_uaspec_system" }, { "type": "string", "name": "edata_uaspec_platform" }, { "type": "string", "name": "edata_uaspec_raw" }, { "type": "string", "name": "edata_state" }, { "type": "array", "name": "edata_props" }, { "type": "string", "name": "edata_prevstate" }, { "type": "string", "name": "edata_dspec_os" }, { "type": "string", "name": "edata_dspec_make" }, { "type": "string", "name": "edata_dspec_id" }, { "type": "long", "name": "edata_dspec_idisk" }, { "type": "long", "name": "edata_dspec_edisk" }, { "type": "long", "name": "edata_dspec_scrn" }, { "type": "string", "name": "edata_dspec_camera" }, { "type": "string", "name": "edata_dspec_cpu" }, { "type": "string", "name": "edata_dspec_webview" }, { "type": "array", "name": "edata_extra_pos" }, { "type": "array", "name": "edata_extra_values" }, { "type": "string", "name": "edata_extra_query" }, { "type": "string", "name": "edata_mode" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "context_rollup_l2" }, { "type": "string", "name": "context_rollup_l3" }, { "type": "array", "name": "context_cdata" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_uid" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_ver" }, { "type": "string", "name": "object_rollup_l1" }, { "type": "string", "name": "object_version" }, { "type": "array", "name": "tags" } ] }, "timestampSpec": { "column": "ets", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_min", "fieldName": "edata_filters_compatibilityLevel_min" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_max", "fieldName": "edata_filters_compatibilityLevel_max" }, { "type": "doubleSum", "name": "edata_dspec_sims", "fieldName": "edata_dspec_sims" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 500000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "invalid_topic", "consumerProperties": { "bootstrap.servers": "kafka-headless.kafka.svc:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.ver", "name": "ver" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.flags.content_denorm", "name": "flags_content_denorm" }, { "type": "path", "expr": "$.flags.coll_denorm", "name": "flags_coll_denorm" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.objectType[*]", "name": "edata_filters_objectType" }, { "type": "path", "expr": "$.edata.filters.version[*]", "name": "edata_filters_version" }, { "type": "path", "expr": "$.edata.filters.status[*]", "name": "edata_filters_status" }, { "type": "path", "expr": "$.edata.filters.id[*]", "name": "edata_filters_id" }, { "type": "path", "expr": "$.edata.filters.isRootOrg", "name": "edata_filters_isRootOrg" }, { "type": "path", "expr": "$.edata.filters.trackable.enabled", "name": "edata_filters_trackable_enabled" }, { "type": "path", "expr": "$.edata.filters.identifier[*]", "name": "edata_filters_identifier" }, { "type": "path", "expr": "$.edata.filters.contentType[*]", "name": "edata_filters_contentType" }, { "type": "path", "expr": "$.edata.filters.mimeType[*]", "name": "edata_filters_mimeType" }, { "type": "path", "expr": "$.edata.filters.hashTagId[*]", "name": "edata_filters_hashTagId" }, { "type": "path", "expr": "$.edata.filters.createdBy", "name": "edata_filters_createdBy" }, { "type": "path", "expr": "$.edata.filters.mediaType[*]", "name": "edata_filters_mediaType" }, { "type": "path", "expr": "$.edata.filters.origin", "name": "edata_filters_origin" }, { "type": "path", "expr": "$.edata.filters.primaryCategory[*]", "name": "edata_filters_primaryCategory" }, { "type": "path", "expr": "$.edata.filters.trackable", "name": "edata_filters_trackable" }, { "type": "path", "expr": "$.edata.sort.lastUpdatedOn", "name": "edata_sort_lastUpdatedOn" }, { "type": "path", "expr": "$.edata.topn[*]", "name": "edata_topn" }, { "type": "path", "expr": "$.edata.pageid", "name": "edata_pageid" }, { "type": "path", "expr": "$.edata.uri", "name": "edata_uri" }, { "type": "path", "expr": "$.edata.subtype", "name": "edata_subtype" }, { "type": "path", "expr": "$.edata.id", "name": "edata_id" }, { "type": "path", "expr": "$.edata.data", "name": "edata_data" }, { "type": "path", "expr": "$.edata.uaspec.agent", "name": "edata_uaspec_agent" }, { "type": "path", "expr": "$.edata.uaspec.ver", "name": "edata_uaspec_ver" }, { "type": "path", "expr": "$.edata.uaspec.system", "name": "edata_uaspec_system" }, { "type": "path", "expr": "$.edata.uaspec.platform", "name": "edata_uaspec_platform" }, { "type": "path", "expr": "$.edata.uaspec.raw", "name": "edata_uaspec_raw" }, { "type": "path", "expr": "$.edata.state", "name": "edata_state" }, { "type": "path", "expr": "$.edata.props[*]", "name": "edata_props" }, { "type": "path", "expr": "$.edata.prevstate", "name": "edata_prevstate" }, { "type": "path", "expr": "$.edata.dspec.os", "name": "edata_dspec_os" }, { "type": "path", "expr": "$.edata.dspec.make", "name": "edata_dspec_make" }, { "type": "path", "expr": "$.edata.dspec.id", "name": "edata_dspec_id" }, { "type": "path", "expr": "$.edata.dspec.idisk", "name": "edata_dspec_idisk" }, { "type": "path", "expr": "$.edata.dspec.edisk", "name": "edata_dspec_edisk" }, { "type": "path", "expr": "$.edata.dspec.scrn", "name": "edata_dspec_scrn" }, { "type": "path", "expr": "$.edata.dspec.camera", "name": "edata_dspec_camera" }, { "type": "path", "expr": "$.edata.dspec.cpu", "name": "edata_dspec_cpu" }, { "type": "path", "expr": "$.edata.dspec.webview", "name": "edata_dspec_webview" }, { "type": "path", "expr": "$.edata.extra.pos[*]", "name": "edata_extra_pos" }, { "type": "path", "expr": "$.edata.extra.values[*]", "name": "edata_extra_values" }, { "type": "path", "expr": "$.edata.extra.query", "name": "edata_extra_query" }, { "type": "path", "expr": "$.edata.mode", "name": "edata_mode" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.context.rollup.l2", "name": "context_rollup_l2" }, { "type": "path", "expr": "$.context.rollup.l3", "name": "context_rollup_l3" }, { "type": "path", "expr": "$.context.cdata[*]", "name": "context_cdata" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.uid", "name": "context_uid" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.ver", "name": "object_ver" }, { "type": "path", "expr": "$.object.rollup.l1", "name": "object_rollup_l1" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.tags[*]", "name": "tags" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.min", "name": "edata_filters_compatibilityLevel_min" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.max", "name": "edata_filters_compatibilityLevel_max" }, { "type": "path", "expr": "$.edata.dspec.sims", "name": "edata_dspec_sims" } ] } }, "appendToExisting": false } } }, "datasource_ref": "sb-telemetry.1_DAY", "retention_period": { "enabled": "false" }, "archival_policy": { "enabled": "false" }, "purge_policy": { "enabled": "false" }, "backup_config": { "enabled": "false" }, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "published_date": "2023-07-03 00:00:00", "metadata": { "aggregated": false, "granularity": "day" } } - public static INVALID_DATASOURCE_REF = { "id": "sb-telemetry_sb-telemetry", "type":"druid", "datasource": "sb-telemetry", "dataset_id": "sb-telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "invalid_datasource", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "eid" }, { "type": "string", "name": "ver" }, { "type": "long", "name": "syncts" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "boolean", "name": "flags_content_denorm" }, { "type": "boolean", "name": "flags_coll_denorm" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "edata_type" }, { "type": "long", "name": "edata_duration" }, { "type": "string", "name": "edata_query" }, { "type": "array", "name": "edata_filters_objectType" }, { "type": "array", "name": "edata_filters_version" }, { "type": "array", "name": "edata_filters_status" }, { "type": "array", "name": "edata_filters_id" }, { "type": "boolean", "name": "edata_filters_isRootOrg" }, { "type": "string", "name": "edata_filters_trackable_enabled" }, { "type": "array", "name": "edata_filters_identifier" }, { "type": "array", "name": "edata_filters_contentType" }, { "type": "array", "name": "edata_filters_mimeType" }, { "type": "array", "name": "edata_filters_hashTagId" }, { "type": "string", "name": "edata_filters_createdBy" }, { "type": "array", "name": "edata_filters_mediaType" }, { "type": "string", "name": "edata_filters_origin" }, { "type": "array", "name": "edata_filters_primaryCategory" }, { "name": "edata_filters_trackable" }, { "type": "string", "name": "edata_sort_lastUpdatedOn" }, { "type": "array", "name": "edata_topn" }, { "type": "string", "name": "edata_pageid" }, { "type": "string", "name": "edata_uri" }, { "type": "string", "name": "edata_subtype" }, { "type": "string", "name": "edata_id" }, { "type": "string", "name": "edata_data" }, { "type": "string", "name": "edata_uaspec_agent" }, { "type": "string", "name": "edata_uaspec_ver" }, { "type": "string", "name": "edata_uaspec_system" }, { "type": "string", "name": "edata_uaspec_platform" }, { "type": "string", "name": "edata_uaspec_raw" }, { "type": "string", "name": "edata_state" }, { "type": "array", "name": "edata_props" }, { "type": "string", "name": "edata_prevstate" }, { "type": "string", "name": "edata_dspec_os" }, { "type": "string", "name": "edata_dspec_make" }, { "type": "string", "name": "edata_dspec_id" }, { "type": "long", "name": "edata_dspec_idisk" }, { "type": "long", "name": "edata_dspec_edisk" }, { "type": "long", "name": "edata_dspec_scrn" }, { "type": "string", "name": "edata_dspec_camera" }, { "type": "string", "name": "edata_dspec_cpu" }, { "type": "string", "name": "edata_dspec_webview" }, { "type": "array", "name": "edata_extra_pos" }, { "type": "array", "name": "edata_extra_values" }, { "type": "string", "name": "edata_extra_query" }, { "type": "string", "name": "edata_mode" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "context_rollup_l2" }, { "type": "string", "name": "context_rollup_l3" }, { "type": "array", "name": "context_cdata" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_uid" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_ver" }, { "type": "string", "name": "object_rollup_l1" }, { "type": "string", "name": "object_version" }, { "type": "array", "name": "tags" } ] }, "timestampSpec": { "column": "ets", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_min", "fieldName": "edata_filters_compatibilityLevel_min" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_max", "fieldName": "edata_filters_compatibilityLevel_max" }, { "type": "doubleSum", "name": "edata_dspec_sims", "fieldName": "edata_dspec_sims" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 500000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "invalid_topic", "consumerProperties": { "bootstrap.servers": "kafka-headless.kafka.svc:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.ver", "name": "ver" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.flags.content_denorm", "name": "flags_content_denorm" }, { "type": "path", "expr": "$.flags.coll_denorm", "name": "flags_coll_denorm" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.objectType[*]", "name": "edata_filters_objectType" }, { "type": "path", "expr": "$.edata.filters.version[*]", "name": "edata_filters_version" }, { "type": "path", "expr": "$.edata.filters.status[*]", "name": "edata_filters_status" }, { "type": "path", "expr": "$.edata.filters.id[*]", "name": "edata_filters_id" }, { "type": "path", "expr": "$.edata.filters.isRootOrg", "name": "edata_filters_isRootOrg" }, { "type": "path", "expr": "$.edata.filters.trackable.enabled", "name": "edata_filters_trackable_enabled" }, { "type": "path", "expr": "$.edata.filters.identifier[*]", "name": "edata_filters_identifier" }, { "type": "path", "expr": "$.edata.filters.contentType[*]", "name": "edata_filters_contentType" }, { "type": "path", "expr": "$.edata.filters.mimeType[*]", "name": "edata_filters_mimeType" }, { "type": "path", "expr": "$.edata.filters.hashTagId[*]", "name": "edata_filters_hashTagId" }, { "type": "path", "expr": "$.edata.filters.createdBy", "name": "edata_filters_createdBy" }, { "type": "path", "expr": "$.edata.filters.mediaType[*]", "name": "edata_filters_mediaType" }, { "type": "path", "expr": "$.edata.filters.origin", "name": "edata_filters_origin" }, { "type": "path", "expr": "$.edata.filters.primaryCategory[*]", "name": "edata_filters_primaryCategory" }, { "type": "path", "expr": "$.edata.filters.trackable", "name": "edata_filters_trackable" }, { "type": "path", "expr": "$.edata.sort.lastUpdatedOn", "name": "edata_sort_lastUpdatedOn" }, { "type": "path", "expr": "$.edata.topn[*]", "name": "edata_topn" }, { "type": "path", "expr": "$.edata.pageid", "name": "edata_pageid" }, { "type": "path", "expr": "$.edata.uri", "name": "edata_uri" }, { "type": "path", "expr": "$.edata.subtype", "name": "edata_subtype" }, { "type": "path", "expr": "$.edata.id", "name": "edata_id" }, { "type": "path", "expr": "$.edata.data", "name": "edata_data" }, { "type": "path", "expr": "$.edata.uaspec.agent", "name": "edata_uaspec_agent" }, { "type": "path", "expr": "$.edata.uaspec.ver", "name": "edata_uaspec_ver" }, { "type": "path", "expr": "$.edata.uaspec.system", "name": "edata_uaspec_system" }, { "type": "path", "expr": "$.edata.uaspec.platform", "name": "edata_uaspec_platform" }, { "type": "path", "expr": "$.edata.uaspec.raw", "name": "edata_uaspec_raw" }, { "type": "path", "expr": "$.edata.state", "name": "edata_state" }, { "type": "path", "expr": "$.edata.props[*]", "name": "edata_props" }, { "type": "path", "expr": "$.edata.prevstate", "name": "edata_prevstate" }, { "type": "path", "expr": "$.edata.dspec.os", "name": "edata_dspec_os" }, { "type": "path", "expr": "$.edata.dspec.make", "name": "edata_dspec_make" }, { "type": "path", "expr": "$.edata.dspec.id", "name": "edata_dspec_id" }, { "type": "path", "expr": "$.edata.dspec.idisk", "name": "edata_dspec_idisk" }, { "type": "path", "expr": "$.edata.dspec.edisk", "name": "edata_dspec_edisk" }, { "type": "path", "expr": "$.edata.dspec.scrn", "name": "edata_dspec_scrn" }, { "type": "path", "expr": "$.edata.dspec.camera", "name": "edata_dspec_camera" }, { "type": "path", "expr": "$.edata.dspec.cpu", "name": "edata_dspec_cpu" }, { "type": "path", "expr": "$.edata.dspec.webview", "name": "edata_dspec_webview" }, { "type": "path", "expr": "$.edata.extra.pos[*]", "name": "edata_extra_pos" }, { "type": "path", "expr": "$.edata.extra.values[*]", "name": "edata_extra_values" }, { "type": "path", "expr": "$.edata.extra.query", "name": "edata_extra_query" }, { "type": "path", "expr": "$.edata.mode", "name": "edata_mode" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.context.rollup.l2", "name": "context_rollup_l2" }, { "type": "path", "expr": "$.context.rollup.l3", "name": "context_rollup_l3" }, { "type": "path", "expr": "$.context.cdata[*]", "name": "context_cdata" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.uid", "name": "context_uid" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.ver", "name": "object_ver" }, { "type": "path", "expr": "$.object.rollup.l1", "name": "object_rollup_l1" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.tags[*]", "name": "tags" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.min", "name": "edata_filters_compatibilityLevel_min" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.max", "name": "edata_filters_compatibilityLevel_max" }, { "type": "path", "expr": "$.edata.dspec.sims", "name": "edata_dspec_sims" } ] } }, "appendToExisting": false } } }, "datasource_ref": "sb-telemetry.1_DAY", "retention_period": { "enabled": "false" }, "archival_policy": { "enabled": "false" }, "purge_policy": { "enabled": "false" }, "backup_config": { "enabled": "false" }, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "published_date": "2023-07-03 00:00:00", "metadata": { "aggregated": false, "granularity": "day" } } - public static MISSING_REQUIRED_FIELDS_CREATE = { "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "telemetry-events", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "fromDate" }, { "type": "string", "name": "toDate" }, { "type": "string", "name": "tes" }, { "type": "string", "name": "uid" }, { "type": "string", "name": "mobile" }, { "type": "string", "name": "ip" }, { "type": "string", "name": "ipv6" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "string", "name": "derivedlocationdata_district" }, { "type": "string", "name": "derivedlocationdata_from" }, { "type": "string", "name": "derivedlocationdata_state" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "type" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "edata_type" }, { "type": "string", "name": "edata_query" }, { "type": "string", "name": "edata_filters_slug" }, { "type": "boolean", "name": "edata_filters_isTenant" }, { "type": "string", "name": "edata_filters_make_type" }, { "type": "string", "name": "edata_topn[*]_id" }, { "name": "edata_items" }, { "type": "array", "name": "userdata_subject" }, { "type": "string", "name": "userdata_district" }, { "type": "string", "name": "userdata_usersubtype" }, { "type": "array", "name": "userdata_grade" }, { "type": "string", "name": "userdata_usersignintype" }, { "type": "string", "name": "userdata_usertype" }, { "type": "string", "name": "userdata_userlogintype" }, { "type": "string", "name": "userdata_state" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "devicedata_statecustomcode" }, { "type": "string", "name": "devicedata_country" }, { "type": "string", "name": "devicedata_iso3166statecode" }, { "type": "string", "name": "devicedata_city" }, { "type": "string", "name": "devicedata_countrycode" }, { "type": "string", "name": "devicedata_state" }, { "type": "string", "name": "devicedata_devicespec_idisk" }, { "type": "string", "name": "devicedata_devicespec_webview" }, { "type": "string", "name": "devicedata_devicespec_os" }, { "type": "string", "name": "devicedata_devicespec_scrn" }, { "type": "string", "name": "devicedata_devicespec_sims" }, { "type": "string", "name": "devicedata_devicespec_cpu" }, { "type": "string", "name": "devicedata_devicespec_id" }, { "type": "string", "name": "devicedata_devicespec_camera" }, { "type": "string", "name": "devicedata_devicespec_edisk" }, { "type": "string", "name": "devicedata_devicespec_make" }, { "type": "string", "name": "devicedata_statecode" }, { "type": "string", "name": "devicedata_districtcustom" }, { "type": "string", "name": "devicedata_statecustomname" }, { "type": "string", "name": "devicedata_userdeclared_district" }, { "type": "string", "name": "devicedata_userdeclared_state" }, { "type": "string", "name": "context_cdata[*]_id" }, { "type": "string", "name": "context_cdata[*]_type" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_version" }, { "type": "string", "name": "ver" } ] }, "timestampSpec": { "column": "arrival-time", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "eid", "fieldName": "eid" }, { "type": "doubleSum", "name": "syncts", "fieldName": "syncts" }, { "type": "doubleSum", "name": "ets", "fieldName": "ets" }, { "type": "doubleSum", "name": "edata_duration", "fieldName": "edata_duration" }, { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_make_vers", "fieldName": "edata_filters_make_vers" }, { "type": "doubleSum", "name": "devicedata_firstaccess", "fieldName": "devicedata_firstaccess" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxRowsPerSegment": 50000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "telemetry", "consumerProperties": {}, "taskCount": 1, "replicas": 1, "taskDuration": "PT8H", "useEarliestOffset": false, "completionTimeout": "PT8H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.make.vers", "name": "edata_filters_make_vers" }, { "type": "path", "expr": "$.devicedata.firstaccess", "name": "devicedata_firstaccess" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.derivedlocationdata.district", "name": "derivedlocationdata_district" }, { "type": "path", "expr": "$.derivedlocationdata.from", "name": "derivedlocationdata_from" }, { "type": "path", "expr": "$.derivedlocationdata.state", "name": "derivedlocationdata_state" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.type", "name": "type" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.slug", "name": "edata_filters_slug" }, { "type": "path", "expr": "$.edata.filters.isTenant", "name": "edata_filters_isTenant" }, { "type": "path", "expr": "$.edata.filters.make.type", "name": "edata_filters_make_type" }, { "type": "path", "expr": "$.edata.topn[*].id", "name": "edata_topn[*]_id" }, { "type": "path", "expr": "$.edata.items", "name": "edata_items" }, { "type": "path", "expr": "$.userdata.subject[*]", "name": "userdata_subject" }, { "type": "path", "expr": "$.userdata.district", "name": "userdata_district" }, { "type": "path", "expr": "$.userdata.usersubtype", "name": "userdata_usersubtype" }, { "type": "path", "expr": "$.userdata.grade[*]", "name": "userdata_grade" }, { "type": "path", "expr": "$.userdata.usersignintype", "name": "userdata_usersignintype" }, { "type": "path", "expr": "$.userdata.usertype", "name": "userdata_usertype" }, { "type": "path", "expr": "$.userdata.userlogintype", "name": "userdata_userlogintype" }, { "type": "path", "expr": "$.userdata.state", "name": "userdata_state" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.devicedata.statecustomcode", "name": "devicedata_statecustomcode" }, { "type": "path", "expr": "$.devicedata.country", "name": "devicedata_country" }, { "type": "path", "expr": "$.devicedata.iso3166statecode", "name": "devicedata_iso3166statecode" }, { "type": "path", "expr": "$.devicedata.city", "name": "devicedata_city" }, { "type": "path", "expr": "$.devicedata.countrycode", "name": "devicedata_countrycode" }, { "type": "path", "expr": "$.devicedata.state", "name": "devicedata_state" }, { "type": "path", "expr": "$.devicedata.devicespec.idisk", "name": "devicedata_devicespec_idisk" }, { "type": "path", "expr": "$.devicedata.devicespec.webview", "name": "devicedata_devicespec_webview" }, { "type": "path", "expr": "$.devicedata.devicespec.os", "name": "devicedata_devicespec_os" }, { "type": "path", "expr": "$.devicedata.devicespec.scrn", "name": "devicedata_devicespec_scrn" }, { "type": "path", "expr": "$.devicedata.devicespec.sims", "name": "devicedata_devicespec_sims" }, { "type": "path", "expr": "$.devicedata.devicespec.cpu", "name": "devicedata_devicespec_cpu" }, { "type": "path", "expr": "$.devicedata.devicespec.id", "name": "devicedata_devicespec_id" }, { "type": "path", "expr": "$.devicedata.devicespec.camera", "name": "devicedata_devicespec_camera" }, { "type": "path", "expr": "$.devicedata.devicespec.edisk", "name": "devicedata_devicespec_edisk" }, { "type": "path", "expr": "$.devicedata.devicespec.make", "name": "devicedata_devicespec_make" }, { "type": "path", "expr": "$.devicedata.statecode", "name": "devicedata_statecode" }, { "type": "path", "expr": "$.devicedata.districtcustom", "name": "devicedata_districtcustom" }, { "type": "path", "expr": "$.devicedata.statecustomname", "name": "devicedata_statecustomname" }, { "type": "path", "expr": "$.devicedata.userdeclared.district", "name": "devicedata_userdeclared_district" }, { "type": "path", "expr": "$.devicedata.userdeclared.state", "name": "devicedata_userdeclared_state" }, { "type": "path", "expr": "$.context.cdata[*].id", "name": "context_cdata[*]_id" }, { "type": "path", "expr": "$.context.cdata[*].type", "name": "context_cdata[*]_type" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.ver", "name": "ver" } ] } }, "appendToExisting": false } } }, "type":"druid", "datasource": "telemetry-events", "datasource_ref": "telemetry-events", "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" };; - public static MISSING_REQUIRED_FIELDS_UPDATE = { "dataset_id": "telemetry", "backup_config": { "enabled": true }, "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z" };; - public static SAMPLE_ID = "telemetry_telemetry-events"; - public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; - public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; - - public static VALID_RECORD = { "dataset_id": "telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "telemetry-events", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "fromDate" }, { "type": "string", "name": "toDate" }, { "type": "string", "name": "tes" }, { "type": "string", "name": "uid" }, { "type": "string", "name": "mobile" }, { "type": "string", "name": "ip" }, { "type": "string", "name": "ipv6" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "string", "name": "derivedlocationdata_district" }, { "type": "string", "name": "derivedlocationdata_from" }, { "type": "string", "name": "derivedlocationdata_state" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "type" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "edata_type" }, { "type": "string", "name": "edata_query" }, { "type": "string", "name": "edata_filters_slug" }, { "type": "boolean", "name": "edata_filters_isTenant" }, { "type": "string", "name": "edata_filters_make_type" }, { "type": "string", "name": "edata_topn[*]_id" }, { "name": "edata_items" }, { "type": "array", "name": "userdata_subject" }, { "type": "string", "name": "userdata_district" }, { "type": "string", "name": "userdata_usersubtype" }, { "type": "array", "name": "userdata_grade" }, { "type": "string", "name": "userdata_usersignintype" }, { "type": "string", "name": "userdata_usertype" }, { "type": "string", "name": "userdata_userlogintype" }, { "type": "string", "name": "userdata_state" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "devicedata_statecustomcode" }, { "type": "string", "name": "devicedata_country" }, { "type": "string", "name": "devicedata_iso3166statecode" }, { "type": "string", "name": "devicedata_city" }, { "type": "string", "name": "devicedata_countrycode" }, { "type": "string", "name": "devicedata_state" }, { "type": "string", "name": "devicedata_devicespec_idisk" }, { "type": "string", "name": "devicedata_devicespec_webview" }, { "type": "string", "name": "devicedata_devicespec_os" }, { "type": "string", "name": "devicedata_devicespec_scrn" }, { "type": "string", "name": "devicedata_devicespec_sims" }, { "type": "string", "name": "devicedata_devicespec_cpu" }, { "type": "string", "name": "devicedata_devicespec_id" }, { "type": "string", "name": "devicedata_devicespec_camera" }, { "type": "string", "name": "devicedata_devicespec_edisk" }, { "type": "string", "name": "devicedata_devicespec_make" }, { "type": "string", "name": "devicedata_statecode" }, { "type": "string", "name": "devicedata_districtcustom" }, { "type": "string", "name": "devicedata_statecustomname" }, { "type": "string", "name": "devicedata_userdeclared_district" }, { "type": "string", "name": "devicedata_userdeclared_state" }, { "type": "string", "name": "context_cdata[*]_id" }, { "type": "string", "name": "context_cdata[*]_type" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_version" }, { "type": "string", "name": "ver" } ] }, "timestampSpec": { "column": "arrival-time", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "eid", "fieldName": "eid" }, { "type": "doubleSum", "name": "syncts", "fieldName": "syncts" }, { "type": "doubleSum", "name": "ets", "fieldName": "ets" }, { "type": "doubleSum", "name": "edata_duration", "fieldName": "edata_duration" }, { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_make_vers", "fieldName": "edata_filters_make_vers" }, { "type": "doubleSum", "name": "devicedata_firstaccess", "fieldName": "devicedata_firstaccess" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxRowsPerSegment": 50000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "telemetry", "consumerProperties": {}, "taskCount": 1, "replicas": 1, "taskDuration": "PT8H", "useEarliestOffset": false, "completionTimeout": "PT8H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.make.vers", "name": "edata_filters_make_vers" }, { "type": "path", "expr": "$.devicedata.firstaccess", "name": "devicedata_firstaccess" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.derivedlocationdata.district", "name": "derivedlocationdata_district" }, { "type": "path", "expr": "$.derivedlocationdata.from", "name": "derivedlocationdata_from" }, { "type": "path", "expr": "$.derivedlocationdata.state", "name": "derivedlocationdata_state" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.type", "name": "type" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.slug", "name": "edata_filters_slug" }, { "type": "path", "expr": "$.edata.filters.isTenant", "name": "edata_filters_isTenant" }, { "type": "path", "expr": "$.edata.filters.make.type", "name": "edata_filters_make_type" }, { "type": "path", "expr": "$.edata.topn[*].id", "name": "edata_topn[*]_id" }, { "type": "path", "expr": "$.edata.items", "name": "edata_items" }, { "type": "path", "expr": "$.userdata.subject[*]", "name": "userdata_subject" }, { "type": "path", "expr": "$.userdata.district", "name": "userdata_district" }, { "type": "path", "expr": "$.userdata.usersubtype", "name": "userdata_usersubtype" }, { "type": "path", "expr": "$.userdata.grade[*]", "name": "userdata_grade" }, { "type": "path", "expr": "$.userdata.usersignintype", "name": "userdata_usersignintype" }, { "type": "path", "expr": "$.userdata.usertype", "name": "userdata_usertype" }, { "type": "path", "expr": "$.userdata.userlogintype", "name": "userdata_userlogintype" }, { "type": "path", "expr": "$.userdata.state", "name": "userdata_state" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.devicedata.statecustomcode", "name": "devicedata_statecustomcode" }, { "type": "path", "expr": "$.devicedata.country", "name": "devicedata_country" }, { "type": "path", "expr": "$.devicedata.iso3166statecode", "name": "devicedata_iso3166statecode" }, { "type": "path", "expr": "$.devicedata.city", "name": "devicedata_city" }, { "type": "path", "expr": "$.devicedata.countrycode", "name": "devicedata_countrycode" }, { "type": "path", "expr": "$.devicedata.state", "name": "devicedata_state" }, { "type": "path", "expr": "$.devicedata.devicespec.idisk", "name": "devicedata_devicespec_idisk" }, { "type": "path", "expr": "$.devicedata.devicespec.webview", "name": "devicedata_devicespec_webview" }, { "type": "path", "expr": "$.devicedata.devicespec.os", "name": "devicedata_devicespec_os" }, { "type": "path", "expr": "$.devicedata.devicespec.scrn", "name": "devicedata_devicespec_scrn" }, { "type": "path", "expr": "$.devicedata.devicespec.sims", "name": "devicedata_devicespec_sims" }, { "type": "path", "expr": "$.devicedata.devicespec.cpu", "name": "devicedata_devicespec_cpu" }, { "type": "path", "expr": "$.devicedata.devicespec.id", "name": "devicedata_devicespec_id" }, { "type": "path", "expr": "$.devicedata.devicespec.camera", "name": "devicedata_devicespec_camera" }, { "type": "path", "expr": "$.devicedata.devicespec.edisk", "name": "devicedata_devicespec_edisk" }, { "type": "path", "expr": "$.devicedata.devicespec.make", "name": "devicedata_devicespec_make" }, { "type": "path", "expr": "$.devicedata.statecode", "name": "devicedata_statecode" }, { "type": "path", "expr": "$.devicedata.districtcustom", "name": "devicedata_districtcustom" }, { "type": "path", "expr": "$.devicedata.statecustomname", "name": "devicedata_statecustomname" }, { "type": "path", "expr": "$.devicedata.userdeclared.district", "name": "devicedata_userdeclared_district" }, { "type": "path", "expr": "$.devicedata.userdeclared.state", "name": "devicedata_userdeclared_state" }, { "type": "path", "expr": "$.context.cdata[*].id", "name": "context_cdata[*]_id" }, { "type": "path", "expr": "$.context.cdata[*].type", "name": "context_cdata[*]_type" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.ver", "name": "ver" } ] } }, "appendToExisting": false } } }, "type":"druid", "datasource": "telemetry-events", "datasource_ref": "telemetry-events", "retention_period": {}, "archival_policy": {}, "purge_policy": {}, "backup_config": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-15T20:45:04.737Z", "updated_date": "2023-03-15T20:45:04.733Z", "published_date": "2023-03-15T20:45:04.733Z", "metadata": { "aggregated": false, "granularity": "day" } } - -} - -class TestDatasetSourceConfig { - public static VALID_SCHEMA = { "connector_type": "kafka", "dataset_id": "observations", "status": DatasetStatus.Live } - public static VALID_UPDATE_SCHEMA = { "id": "observations_kafka", "connector_type": "kafka", "dataset_id": "observations", "status": DatasetStatus.Retired } - public static INVALID_SCHEMA = { "connector_type": "kafka", "dataset_id": "observations", "status": {} } - public static MISSING_REQUIRED_FIELDS_CREATE = { "connector_type": "kafka", "status": DatasetStatus.Live } - public static SAMPLE_ID = "observations_kafka"; - public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; - public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; - public static MISSING_REQUIRED_FIELDS_UPDATE = { "connector_type": "kafka", "status": DatasetStatus.Live } - public static VALID_RECORD = { "connector_type": "kafka", "dataset_id": "observations", "connector_config": {}, "status": DatasetStatus.Live, "connector_stats": {}, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-04-07T18:30:00.000Z", "updated_date": "2023-04-07T18:30:00.000Z" } -} - -class TestExhaust { - public static INVALID_DATE_RANGE = { "from": "20213-06-21", "to": "1234-12-12", "type": "transformed" }; - public static DATE_RANGE_OVER_LIMIT = { "from": "2023-06-01", "to": "2023-07-03", "type": "transformed" }; - public static VALID_REQUEST = { "from": "2023-06-01", "to": "2023-06-30", "type": "transformed" }; -} - - class TestSubmitIngestion { - public static VALID_INGESTION_SPEC = { - "type": "kafka", - "spec": { - "dataSchema": { - "dataSource": "obsrv-telemetry-events", - "dimensionsSpec": { - "dimensions": [] - }, - "timestampSpec": { - "column": "arrival-time", - "format": "auto" - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": "HOUR", - "rollup": false - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 50000, - "logParseExceptions": true - }, - "ioConfig": { - "type": "kafka", - "topic": "obsrv.telemetry.input", - "consumerProperties": {"bootstrap.servers": "localhost:9092"}, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT8H", - "useEarliestOffset": false, - "completionTimeout": "PT8H", - "inputFormat": { - "type": "json", - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [] - } - }, - "appendToExisting": false - } - } - } - public static INVALID_INGESTION_SPEC = { - "type": "kafka", - "spec": "ingestion_spec" - } -} - -export { TestDruidQuery, TestDataIngestion, TestDataset, TestDataSource, TestDatasetSourceConfig, TestExhaust, TestSubmitIngestion}; \ No newline at end of file diff --git a/api-service/src/v1/test/IngestorTestService.spec.ts b/api-service/src/v1/test/IngestorTestService.spec.ts deleted file mode 100644 index 855cf0c5..00000000 --- a/api-service/src/v1/test/IngestorTestService.spec.ts +++ /dev/null @@ -1,276 +0,0 @@ -import app from "../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { TestDataIngestion, TestSubmitIngestion } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import constants from "../resources/Constants.json" -import { dbConnector, globalCache, ingestorService, kafkaConnector } from "../routes/Router"; -import { describe, it } from 'mocha'; -import nock from "nock"; -import { DatasetStatus } from "../models/DatasetModels"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("DATA INGEST API", () => { - afterEach(() => { - chai.spy.restore() - }) - - it("it should ingest data successfully", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" }, "extraction_config": { "is_batch_event": true, "extraction_key": "events", "batch_id": "id" } } ] - }) - chai.spy.on(kafkaConnector.telemetryService, "dispatch", () => { - return Promise.resolve("data ingested") - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should ingest data successfully for batch even for individual events", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" }, "extraction_config": { "is_batch_event": true, "extraction_key": "events", "batch_id": "id" } } ] - }) - chai.spy.on(kafkaConnector.telemetryService, "dispatch", () => { - return Promise.resolve("data ingested") - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INDIVIDUAL_EVENT) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should not ingest data successfully when kafka is unable to connect", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" } } ] - }) - chai.spy.on(kafkaConnector.telemetryService, "dispatch", () => { - return Promise.reject("error connecting to kafka") - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should not ingest data when invalid extraction config present for batch", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" }, "extraction_config": { "is_batch_event": true, "extraction_key": "eventas", "batch_id": "ids" } } ] - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should not ingest data successfully", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": "datasetId", "dataset_config": { "entry_topic": "topic" } } ] - }) - - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - done() - }) - }); - it("it should not ingest data successfully", (done) => { - chai.spy.on(globalCache, 'get', () => { - return [ {} ] - }) - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "404_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(globalCache, "get") - chai.spy.restore(dbConnector, "listRecords") - done(); - }); - }); - it("it should not ingest data when datasetid param is empty", (done) => { - chai - .request(app) - .post(config.apiDatasetIngestEndPoint.replace(':datasetId', " /")) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); - it("it should not establish connection with kafka", (done) => { - chai.spy.on(kafkaConnector.telemetryService, "health", () => { - return Promise.reject("error connecting to kafka") - }) - ingestorService.init() - expect(kafkaConnector.telemetryService.health).to.be.called - chai.spy.restore(kafkaConnector.telemetryService, "health") - done(); - }); - it("it should establish connection with kafka", (done) => { - chai.spy.on(kafkaConnector.telemetryService, "health", () => { - return Promise.resolve("connected to kafka") - }) - ingestorService.init() - expect(kafkaConnector.telemetryService.health).to.be.called - chai.spy.restore(kafkaConnector.telemetryService, "health") - done(); - }); -}) - -describe("SUBMIT INGESTION API", () => { - beforeEach(() => { - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSubmitIngestionEndPoint) - .reply(200) - }) - afterEach(() => { - nock.cleanAll() - }) - it("should submit ingestion successfully", (done)=>{ - chai - .request(app) - .post(config.apiSubmitIngestionEndPoint) - .send(TestSubmitIngestion.VALID_INGESTION_SPEC) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.submit_ingestion.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - done() - }) - }) - it("should throw error for invalid request object", (done)=>{ - chai - .request(app) - .post(config.apiSubmitIngestionEndPoint) - .send(TestSubmitIngestion.INVALID_INGESTION_SPEC) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.submit_ingestion.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done() - }) - }) -}) - -describe("SUBMIT INGESTION ERROR SCENARIOS", ()=>{ - beforeEach(() => { - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSubmitIngestionEndPoint) - .reply(500) - }) - afterEach(() => { - nock.cleanAll() - }) - it("should handle errors", (done)=>{ - chai - .request(app) - .post(config.apiSubmitIngestionEndPoint) - .send(TestSubmitIngestion.VALID_INGESTION_SPEC) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.submit_ingestion.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done() - }) -}) -}) diff --git a/api-service/src/v1/test/QueryTestService.spec.ts b/api-service/src/v1/test/QueryTestService.spec.ts deleted file mode 100644 index bdfb6d60..00000000 --- a/api-service/src/v1/test/QueryTestService.spec.ts +++ /dev/null @@ -1,738 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import nock from "nock"; -import httpStatus from "http-status"; -import { TestDruidQuery } from "./Fixtures"; -import { config } from "./Config"; -import constants from "../resources/Constants.json"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { QueryValidator } from "../validators/QueryValidator"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; -import * as lakehouseutil from "../helpers/LakehouseUtil"; -chai.use(chaiSpies) -chai.should(); -chai.use(chaiHttp); - -describe("QUERY API", () => { - - describe("If service is down", () => { - - it("it should raise error when native query endpoint is called", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(500) - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - nock.cleanAll(); - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("should raise error when sql query endpoint is called", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(500) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - nock.cleanAll(); - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - }); - describe("POST /query/v2/native-query", () => { - it("it should fetch information from druid data source", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query, when datarange given as list is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_DATE_RANGE_GIVEN_AS_LIST)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query, when datarange given as string is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_DATE_RANGE_GIVEN_AS_STRING)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should set threshold to default when given threshold is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_THRESHOLD_QUERY)) // given threshold is 1K - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should set row_limit to default when given row_limit is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_LIMIT_QUERY)) // given row_limit is 1K - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should set threshold to default when threshold is not given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_THRESOLD_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query when date range is not given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_DATE_RANGE_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should skip validation and allow druid for query if rules does not exist for datasource", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.UNSUPPORTED_DATA_SOURCE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should skip validation", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.SKIP_VALIDATION_NATIVE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query with unsupported schema", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.UNSUPPORTED_SCHEMA)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject the query because of incorrect url", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .get("/invalid/endpoint") - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.default.api_id); - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - }); - describe("POST /druid/v2/sql", () => { - it("it should allow druid to query when a valid sql query is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should update row_limit to default when row_limit is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_LIMIT_SQL_QUERY)) // given row_limit is 1K - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should set row_limit to default when none is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_LIMIT_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should reject the query when daterange range is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_DATE_RANGE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should reject the query when no daterange is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_DATE_RANGE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should skip validation and allow druid for query if rules does not exist for datasource ", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.UNSUPPORTED_DATASOURCE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }) - }) - it("it should skip validation", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.SKIP_VALIDATION_SQL)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }) - }) - it("should throw error for invalid sql query", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, ["sample_ref"]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [{ events: [] }]); - chai. - request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.INVALID_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords"); - nock.cleanAll(); - done(); - }) - }) - it("should throw error is table name is missing from the SQL Query", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, ["sample_ref"]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [{ events: [] }]); - chai. - request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.MISSING_TABLE_NAME)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords"); - nock.cleanAll(); - done(); - }) - }) - }) - describe("error scenarios", () => { - it("should handle the error", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - throw new Error("error occured while fetching records") - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [{ events: [] }]); - - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }) - }) - }) - it("should not validate if called with invalid url", async () => { - const queryValidator = new QueryValidator() - await queryValidator.validate({}, "obsrv.api") - .then((result) => { - result.isValid.should.be.equal(false) - }) - }), - describe("Datalake query API POST /data/v1/sql-query", () => { - - it("it should retrieve data from datalake when a valid sql query is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - return Promise.resolve([{ totalRatingsCount: 10 }]) - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY_DATALAKE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }); - }) - it("it should handle error scenarios in datalake query", (done)=>{ - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - throw new Error("error occured while fetching records") - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY_DATALAKE)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }) - }) - it("should throw error incase of invalid datasource type", (done)=>{ - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.INVALID_DATASOURCE_TYPE)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - done(); - }) - }) - it("should throw error incase of empty sql query", (done)=>{ - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.EMPTY_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - - done(); - }) - }) - it("should handle native scenarios in case of trino query", (done)=>{ - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - return Promise.resolve([{ totalRatingsCount: 10 }]) - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY_DATALAKE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }); - }) - it("should consider sql query as case insensitive", (done)=>{ - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - return Promise.resolve([{ totalRatingsCount: 10 }]) - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.CASE_INSENSITIVE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }); - }) - }) -}) \ No newline at end of file diff --git a/api-service/src/v1/test/TestConnector.spec.ts b/api-service/src/v1/test/TestConnector.spec.ts deleted file mode 100644 index 64cdaf21..00000000 --- a/api-service/src/v1/test/TestConnector.spec.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { DbConnector } from "../connectors/DbConnector"; -import { KafkaConnector } from "../connectors/KafkaConnector"; -import { HTTPConnector } from "../connectors/HttpConnector"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import { describe, it } from 'mocha'; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const dbConnectorConfig = { - client: "postgresql", - connection: { - host: 'localhost', - port: 5432, - database: 'test', - user: 'test', - password: 'test', - } -} - -describe("Testing Db connector", () => { - it("should successfully arrange connection with database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "select", () => { - return Promise.resolve() - }) - dbConnector.init() - chai.expect(dbConnector.pool.select).to.be.called - chai.spy.restore(dbConnector.pool, "select") - done() - }) - it("should not connect to database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "connect", () => { - return Promise.reject("error occurred while connecting to postgres") - }) - dbConnector.init() - chai.expect(dbConnector.connect).to.be.called - chai.spy.restore(dbConnector, "connect") - done() - }) - it("should not insert records into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "insertRecord", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("insert", { "table": "users", "fields": {} }) - chai.expect(dbConnector.insertRecord).to.be.called - chai.spy.restore(dbConnector, "insertRecord") - done() - }) - it("should insert record into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "insertRecord", () => { - return Promise.resolve([]) - }) - dbConnector.execute("insert", { "table": "users", "fields": {} }) - chai.expect(dbConnector.insertRecord).to.be.called - chai.spy.restore(dbConnector, "insertRecord") - done() - }) - it("should not update records in database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "updateRecord", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).to.be.called - chai.spy.restore(dbConnector, "updateRecord") - done() - }) - it("should update records in database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "updateRecord", () => { - return Promise.resolve([]) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).to.be.called - chai.spy.restore(dbConnector, "updateRecord") - done() - }) - it("should throw error while updating records into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "transaction", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).throw - chai.spy.restore(dbConnector.pool, "transaction") - done() - }) - it("should not throw error while updating records into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "transaction", () => { - return (Promise.resolve([])) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).to.not.throw - chai.spy.restore(dbConnector.pool, "transaction") - done() - }) - it("should not retrieve records from database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "readRecords", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("read", { "table": "users", "fields": {} }) - chai.expect(dbConnector.readRecords).to.be.called - chai.spy.restore(dbConnector, "readRecords") - done() - }) - it("should retrieve records from database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "readRecords", () => { - return Promise.resolve([]) - }) - dbConnector.execute("read", { "table": "users", "fields": {} }) - chai.expect(dbConnector.readRecords).to.be.called - chai.spy.restore(dbConnector, "readRecords") - done() - }) - it("should not disconnect from the database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "destroy", () => { - return Promise.reject(new Error("error occurred while disconnecting from postgres")) - }) - dbConnector.close() - chai.expect(dbConnector.pool.destroy).to.throw - chai.spy.restore(dbConnector.pool, "destroy") - done() - }) - it("should disconnect from the database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "destroy", () => { - return Promise.resolve() - }) - dbConnector.close() - chai.expect(dbConnector.pool.destroy).to.not.throw - chai.spy.restore(dbConnector.pool, "destroy") - done() - }) - it("should list records", (done)=>{ - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "select", () => { - return Promise.resolve() - }) - dbConnector.listRecords('datasets') - chai.expect(dbConnector.pool.select).to.not.throw - chai.spy.restore(dbConnector.pool, "select") - done() - }) -}) -describe('testing httpConnector', () => { - describe('execute', () => { - it('should throw an error', (done) => { - const sampleHttpInstance = new HTTPConnector('example.com'); - const spy = chai.spy.on(sampleHttpInstance, 'execute'); - (() => sampleHttpInstance.execute('sample string')).should.throw(Error); - spy.should.have.been.called.once; - chai.spy.restore(sampleHttpInstance, 'execute') - done() - }); - }); - - describe('close', () => { - it('should throw an error', (done) => { - const sampleHttpInstance = new HTTPConnector('example.com'); - const spy = chai.spy.on(sampleHttpInstance, 'close'); - (() => sampleHttpInstance.close()).should.throw(Error); - spy.should.have.been.called.once; - chai.spy.restore(sampleHttpInstance, 'execute') - done() - }); - }); - -}) -describe("testing kafkaConnector", () => { - describe('close', () => { - it('should throw an error', (done) => { - const sampleKafkaInstance = new KafkaConnector(); - const spy = chai.spy.on(sampleKafkaInstance, 'close'); - (() => sampleKafkaInstance.close()).should.throw(Error); - spy.should.have.been.called.once; - chai.spy.restore(sampleKafkaInstance, 'execute') - done() - }); - }); - -}) \ No newline at end of file diff --git a/api-service/src/v1/utils/common.ts b/api-service/src/v1/utils/common.ts deleted file mode 100644 index eed6006d..00000000 --- a/api-service/src/v1/utils/common.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { DateRange } from "../models/ExhaustModels"; -import moment from "moment"; -import * as _ from "lodash"; -import { Request, Response } from "express"; - -export const getPeriodInterval = (since: String): DateRange => { - const period = since.toLowerCase().split("_")[1]; - return { - from: moment().subtract(period, "days").format("yyyy-MM-DD"), - to: moment().startOf("day").format("yyyy-MM-DD"), - }; -}; - -export const getDateRange = (request: Request, response: Response) => { - const { from, to, since } = request.query; - let period; - if (!_.isUndefined(since)) { - period = getPeriodInterval(since.toString()); - } else if (from != undefined && to != undefined) { - period = { from: from.toString(), to: to.toString() }; - } else { - period = getPeriodInterval("LAST_0_DAYS"); - } - return period; -}; - -export const getFileKey = (key: string): string => { - let keyArray: string[] = key.split("/"); - return keyArray[keyArray.length - 1].slice(0, 10); -}; - -export const isValidDateRange = ( - fromDate: moment.Moment, toDate: moment.Moment, allowedRange: number = 0 -): boolean => { - const differenceInDays = Math.abs(fromDate.diff(toDate, "days")); - const isValidDates = differenceInDays > allowedRange ? false : true; - return isValidDates; -}; diff --git a/api-service/src/v1/validators/QueryValidator.ts b/api-service/src/v1/validators/QueryValidator.ts deleted file mode 100644 index a49f2a48..00000000 --- a/api-service/src/v1/validators/QueryValidator.ts +++ /dev/null @@ -1,228 +0,0 @@ -import httpStatus from "http-status"; -import _ from "lodash"; -import moment, { Moment } from "moment"; -import { queryRules } from "../configs/QueryRules"; -import { IConnector, IValidator } from "../models/DatasetModels"; -import { ICommonRules, ILimits, IQuery, IQueryTypeRules, IRules } from "../models/QueryModels"; -import { ValidationStatus } from "../models/ValidationModels"; -import constants from "../resources/Constants.json"; -import { dbConnector } from "../routes/Router"; -import { routesConfig } from "../configs/RoutesConfig"; -import { config } from "../configs/Config"; -import { isValidDateRange } from "../utils/common"; -import { HTTPConnector } from "../connectors/HttpConnector"; -export class QueryValidator implements IValidator { - private limits: ILimits; - private momentFormat: string; - private httpConnector: any - constructor() { - this.limits = queryRules - this.momentFormat = "YYYY-MM-DD HH:MI:SS" - this.httpConnector = new HTTPConnector(`${config.query_api.druid.host}:${config.query_api.druid.port}`).connect() - } - public async validate(data: any, id: string): Promise { - let validationStatus, dataSource, shouldSkip; - switch (id) { - case routesConfig.query.native_query.api_id: - validationStatus = await this.validateNativeQuery(data) - dataSource = this.getDataSource(data) - shouldSkip = _.includes(config.exclude_datasource_validation, dataSource); - return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus - case routesConfig.query.sql_query.api_id: - validationStatus = await this.validateSqlQuery(data) - if (validationStatus.isValid) { - dataSource = this.getDataSource(data) - shouldSkip = _.includes(config.exclude_datasource_validation, dataSource); - return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus - } - return validationStatus - default: - return { isValid: false } - } - } - - private validateNativeQuery(data: any): ValidationStatus { - let queryObj: IQuery = data; - this.setQueryLimits(data, this.limits.common); - let dataSourceLimits = this.getDataSourceLimits(this.getDataSource(data)); - try { - return (!_.isEmpty(dataSourceLimits)) ? this.validateQueryRules(queryObj, dataSourceLimits.queryRules[queryObj.query.queryType as keyof IQueryTypeRules]) : { isValid: true } - } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while validating native query", code: error.code || httpStatus["400_NAME"] }; - } - } - - private validateSqlQuery(data: IQuery): ValidationStatus { - try { - let query = data.querySql.query; - if (_.isEmpty(query)) { - return { isValid: false, message: "Query must not be empty", code: httpStatus["400_NAME"] }; - } - const fromClause = /\bFROM\b/i; - const isFromClausePresent = fromClause.test(query) - if (!isFromClausePresent) { - return { isValid: false, message: "Invalid SQL Query", code: httpStatus["400_NAME"] }; - } - const fromIndex = query.search(fromClause); - const dataset = query.substring(fromIndex + 4).trim().split(/\s+/)[0].replace(/\\/g, ""); - if (_.isEmpty(dataset)) { - return { isValid: false, message: "Dataset name must be present in the SQL Query", code: httpStatus["400_NAME"] }; - } - this.setQueryLimits(data, this.limits.common); - let datasource = this.getDataSource(data); - let dataSourceLimits = this.getDataSourceLimits(datasource); - return (!_.isEmpty(dataSourceLimits)) ? this.validateQueryRules(data, dataSourceLimits.queryRules.scan) : { isValid: true }; - } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while validating SQL query", code: error.code || httpStatus["500_NAME"] }; - } - } - - private validateQueryRules(queryPayload: IQuery, limits: IRules): ValidationStatus { - let fromDate: Moment | undefined, toDate: Moment | undefined; - let allowedRange = limits.maxDateRange; - if (queryPayload.query) { - const dateRange = this.getIntervals(queryPayload.query); - const extractedDateRange = Array.isArray(dateRange) ? dateRange[0].split("/") : dateRange.toString().split("/"); - fromDate = moment(extractedDateRange[0], this.momentFormat); - toDate = moment(extractedDateRange[1], this.momentFormat); - } else { - let query = queryPayload.querySql.query; - query = query.toUpperCase().replace(/\s+/g, " ").trim(); - let vocabulary = query.split(/\s+/); - let fromDateIndex = vocabulary.indexOf("TIMESTAMP"); - let toDateIndex = vocabulary.lastIndexOf("TIMESTAMP"); - fromDate = moment(vocabulary[fromDateIndex + 1], this.momentFormat); - toDate = moment(vocabulary[toDateIndex + 1], this.momentFormat); - } - const isValidDates = fromDate && toDate && fromDate.isValid() && toDate.isValid() - return isValidDates ? this.validateDateRange(fromDate, toDate, allowedRange) - : { isValid: false, message: constants.ERROR_MESSAGE.NO_DATE_RANGE, code: httpStatus["400_NAME"] }; - }; - - private getDataSource(queryPayload: IQuery): string { - if (queryPayload.querySql) { - let query = queryPayload.querySql.query; - query = query.replace(/\s+/g, " ").trim(); - const fromIndex = query.search(/\bFROM\b/i); - const dataSource = query.substring(fromIndex).split(/\s+/)[1].replace(/\\/g, "").replace(/"/g, ""); - return dataSource; - } else { - const dataSourceField: any = queryPayload.query.dataSource - if (typeof dataSourceField == 'object') { return dataSourceField.name } - return dataSourceField - } - }; - - private getDataSourceLimits(datasource: string): any { - for (var index = 0; index < this.limits.rules.length; index++) { - if (this.limits.rules[index].dataset == datasource) { - return this.limits.rules[index]; - } - } - }; - - private validateDateRange(fromDate: moment.Moment, toDate: moment.Moment, allowedRange: number = 0): ValidationStatus { - const isValidDates = isValidDateRange(fromDate, toDate, allowedRange); - return isValidDates - ? { isValid: true, code: httpStatus[200] } - : { isValid: false, message: constants.ERROR_MESSAGE.INVALID_DATE_RANGE.replace("${allowedRange}", allowedRange.toString()), code: httpStatus["400_NAME"] }; - }; - - private getLimit(queryLimit: number, maxRowLimit: number) { - return queryLimit > maxRowLimit ? maxRowLimit : queryLimit; - }; - - private setQueryLimits(queryPayload: IQuery, limits: ICommonRules) { - if (queryPayload.query) { - if (queryPayload.query.threshold) { - queryPayload.query.threshold = this.getLimit(queryPayload.query.threshold, limits.maxResultThreshold); - } else { - queryPayload.query.threshold = limits.maxResultThreshold; - } - if (queryPayload.query.limit) { - queryPayload.query.limit = this.getLimit(queryPayload.query.limit, limits.maxResultRowLimit); - } else { - queryPayload.query.limit = limits.maxResultRowLimit; - } - } else { - const limitClause = /\bLIMIT\b/i; - const vocabulary = queryPayload.querySql.query.split(/\s+/); // Splitting the query by whitespace - const queryLimitIndex = vocabulary.findIndex(word => limitClause.test(word)); - const queryLimit = Number(vocabulary[queryLimitIndex + 1]); - - if (isNaN(queryLimit)) { - // If "LIMIT" clause doesn't exist or its value is not a number, update the query - const updatedVocabulary = [...vocabulary, "LIMIT", limits.maxResultRowLimit]; - queryPayload.querySql.query = updatedVocabulary.join(" "); - } else { - // If "LIMIT" clause exists and its value is a number, update the limit - const newLimit = this.getLimit(queryLimit, limits.maxResultRowLimit); - vocabulary[queryLimitIndex + 1] = newLimit.toString(); - queryPayload.querySql.query = vocabulary.join(" "); - } - } - } - public async validateDatasource(datasource: any) { - let existingDatasources = await this.httpConnector(config.query_api.druid.list_datasources_path, {}) - if (!_.includes(existingDatasources.data, datasource)) { - let error = constants.INVALID_DATASOURCE - error.message = error.message.replace('${datasource}', datasource) - throw error - } - return - } - public async setDatasourceRef(dataSource: string, payload: any): Promise { - try { - const granularity = _.get(payload, 'context.granularity') - const dataSourceType = _.get(payload, 'context.dataSourceType', config.query_api.druid.queryType) - let dataSourceRef = await this.getDataSourceRef(dataSource, granularity, dataSourceType); - if (dataSourceType === config.query_api.druid.queryType) await this.validateDatasource(dataSourceRef) - if (payload?.querySql && dataSourceType === config.query_api.druid.queryType) { - payload.querySql.query = payload.querySql.query.replace(dataSource, dataSourceRef) - } - else if (payload?.querySql && dataSourceType === config.query_api.lakehouse.queryType) { - // hudi tables doesn't support table names contain '-' so we need to replace it with '_' - let modifiedDataSource = dataSourceRef.replace(/"/g, "").replace(/-/g, "_") - payload.querySql.query = payload.querySql.query.replace(dataSource, modifiedDataSource) - } - else { - payload.query.dataSource = dataSourceRef - } - return { isValid: true }; - } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while fetching datasource record", code: error.code || httpStatus["400_NAME"] }; - } - } - - public async getDataSourceRef(datasource: string, granularity: string | undefined, dataSourceType: string): Promise { - let storageType = dataSourceType === config.query_api.lakehouse.queryType ? config.datasource_storage_types.datalake : config.datasource_storage_types.druid - const records: any = await dbConnector.readRecords("datasources", { "filters": { "dataset_id": datasource, "type": storageType } }) - if (records.length == 0) { - const error = { ...constants.INVALID_DATASOURCE } - error.message = error.message.replace('${datasource}', datasource) - throw error - } - if (storageType === config.datasource_storage_types.datalake) return `${config.query_api.lakehouse.catalog}.${config.query_api.lakehouse.schema}.${records[0].datasource_ref}_ro` - const record = records.filter((record: any) => { - const aggregatedRecord = _.get(record, "metadata.aggregated") - if (granularity) - return aggregatedRecord && _.get(record, "metadata.granularity") === granularity; - else - return !aggregatedRecord - }); - - if (record.length == 0) { - const error = { ...constants.INVALID_DATASOURCE } - error.message = error.message.replace('${datasource}', datasource) - if (granularity !== undefined) { - error.message = `Aggregate ${error.message}` - } - throw error - } - return record[0].datasource_ref - } - - private getIntervals(payload: any) { - return (typeof payload.intervals == 'object' && !Array.isArray(payload.intervals)) ? payload.intervals.intervals : payload.intervals - } -} diff --git a/api-service/src/v1/validators/RequestsValidator.ts b/api-service/src/v1/validators/RequestsValidator.ts deleted file mode 100644 index 871c59ad..00000000 --- a/api-service/src/v1/validators/RequestsValidator.ts +++ /dev/null @@ -1,87 +0,0 @@ -import Ajv from "ajv"; -import addFormats from "ajv-formats"; -import fs from "fs"; -import httpStatus from "http-status"; -import { IValidator } from "../models/DatasetModels"; -import { ValidationStatus } from "../models/ValidationModels"; -import { routesConfig } from "../configs/RoutesConfig"; - -export class RequestsValidator implements IValidator { - private schemaBasePath: string = "/src/v1/resources/"; - private reqSchemaMap = new Map(); - private validator: Ajv; - - constructor() { - this.validator = new Ajv(); - addFormats(this.validator); - this.loadSchemas(); - } - - validate(data: any, id: string): ValidationStatus { - return this.validateRequest(data, id); - } - - validateQueryParams(data: any, id: string): ValidationStatus { - return this.validateRequestParams(data, id); - } - - private validateRequest(data: any, id: string): ValidationStatus { - let validRequestObj = this.validator.validate(this.getReqSchema(id), data); - if (!validRequestObj) { - let error = this.validator.errors; - let errorMessage = error![0].instancePath.replace("/", "") + " " + error![0].message; - return { error: httpStatus["400_NAME"], isValid: false, message: errorMessage, code: httpStatus["400_NAME"] }; - } else { - return { isValid: true, message: "Validation Success", code: httpStatus[200] }; - } - }; - - private validateRequestParams(data: any, id: string): ValidationStatus { - let validRequestObj = this.validator.validate(this.getReqSchema(id), data); - if (!validRequestObj) { - let error = this.validator.errors; - const property = error![0].instancePath.replace("/", ""); - let errorMessage = `property \"${property}\"` + " " + error![0].message; - return { error: httpStatus["400_NAME"], isValid: false, message: errorMessage, code: httpStatus["400_NAME"] }; - } else { - return { isValid: true, message: "Validation Success", code: httpStatus[200] }; - } - }; - - private schemasMapping(): Record { - return [ - routesConfig.query.native_query, - routesConfig.query.sql_query, - routesConfig.data_ingest, - routesConfig.tenant_ingest, - routesConfig.config.dataset.save, - routesConfig.config.datasource.save, - routesConfig.config.dataset.list, - routesConfig.config.datasource.list, - routesConfig.config.dataset.update, - routesConfig.config.datasource.update, - routesConfig.config.dataset_source_config.save, - routesConfig.config.dataset_source_config.update, - routesConfig.config.dataset_source_config.list, - routesConfig.exhaust, - routesConfig.submit_ingestion, - routesConfig.schema_validator - ] - } - - private loadSchemas(): void { - this.schemasMapping().map((routeObject: Record) => - this.reqSchemaMap.set(routeObject.api_id, this.loadSchemaFromFile(routeObject.validation_schema)) - ) - } - - private loadSchemaFromFile(filename: string): any { - const filePath = process.cwd() + `${this.schemaBasePath}schemas/` + filename; - const fileContent = fs.readFileSync(filePath, "utf8"); - return JSON.parse(fileContent); - } - - private getReqSchema(apiId: string): any { - return this.reqSchemaMap.get(apiId); - } -} diff --git a/api-service/src/v2/configs/DatasetConfigDefault.ts b/api-service/src/v2/configs/DatasetConfigDefault.ts deleted file mode 100644 index 52016f38..00000000 --- a/api-service/src/v2/configs/DatasetConfigDefault.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { config } from "./Config"; -import { DatasetStatus, ValidationMode } from "../types/DatasetModels"; - -export const defaultMasterConfig = { - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "denorm_config": { - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "denorm_fields": [] - }, - "router_config": { - "topic": "" - }, - "tags": [], - "dataset_config": { - "data_key": "", - "timestamp_key": "", - "entry_topic": config.telemetry_service_config.kafka.topics.createMasterDataset, - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "index_data": true, - "redis_db": 3, - "file_upload_path": [] - }, - "status": DatasetStatus.Draft, - "version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" -} - -export const defaultDatasetConfig = { - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "denorm_config": { - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "denorm_fields": [] - }, - "router_config": { - "topic": "" - }, - "tags": [], - "dataset_config": { - "data_key": "", - "timestamp_key": "", - "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "index_data": true, - "redis_db": 0, - "file_upload_path": [] - }, - "status": DatasetStatus.Draft, - "version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" -} - -export const validDatasetFields = ["dataset_id", "id", "name", "type", "validation_config", "extraction_config", "dedup_config", "data_schema", "router_config", "denorm_config", "transformations_config", "dataset_config", "tags", "status", "version", "created_by", "updated_by", "created_date", "updated_date", "published_date", "version_key"] \ No newline at end of file diff --git a/api-service/src/v2/configs/IngestionConfig.ts b/api-service/src/v2/configs/IngestionConfig.ts deleted file mode 100644 index 233b9c17..00000000 --- a/api-service/src/v2/configs/IngestionConfig.ts +++ /dev/null @@ -1,17 +0,0 @@ - -const env = process.env; - -export const ingestionConfig = { - "indexCol": { "Event Arrival Time": "obsrv_meta.syncts" }, - "granularitySpec": { - "rollup": false, - "segmentGranularity": env.segment_granularity || "DAY" - }, - "ioconfig": { "topic": "", "taskDuration": "PT1H" }, - "tuningConfig": { "maxRowPerSegment": 5000000, "taskCount": 1 }, - "query_granularity": "none", - "use_earliest_offset": true, - "completion_timeout": "PT1H", - "maxBytesInMemory": env.max_bytes_in_memory || 134217728, - "syncts_path": "$.obsrv_meta.syncts", -} diff --git a/api-service/src/v2/configs/QueryRules.ts b/api-service/src/v2/configs/QueryRules.ts deleted file mode 100644 index be5709cc..00000000 --- a/api-service/src/v2/configs/QueryRules.ts +++ /dev/null @@ -1,56 +0,0 @@ -const maxDateRange = process.env.max_date_range ? parseInt(process.env.max_date_range) : 30 // in days - -export const queryRules = { - "common": { - "maxResultThreshold": process.env.max_query_threshold ? parseInt(process.env.max_query_threshold) : 5000, - "maxResultRowLimit": process.env.max_query_limit ? parseInt(process.env.max_query_limit) : 5000 - }, - "rules": [ - { - "dataset": "telemetry-events", - "queryRules": { - "groupBy": { - "maxDateRange": maxDateRange - }, - "scan": { - "maxDateRange": maxDateRange - }, - "search": { - "maxDateRange": maxDateRange - }, - "timeBoundary": { - "maxDateRange": maxDateRange - }, - "timeseries": { - "maxDateRange": maxDateRange - }, - "topN": { - "maxDateRange": maxDateRange - } - } - }, - { - "dataset": "summary-events", - "queryRules": { - "groupBy": { - "maxDateRange": maxDateRange - }, - "scan": { - "maxDateRange": maxDateRange - }, - "search": { - "maxDateRange": maxDateRange - }, - "timeBoundary": { - "maxDateRange": maxDateRange - }, - "timeseries": { - "maxDateRange": maxDateRange - }, - "topN": { - "maxDateRange": maxDateRange - } - } - } - ] -} diff --git a/api-service/src/v2/connections/commandServiceConnection.ts b/api-service/src/v2/connections/commandServiceConnection.ts deleted file mode 100644 index 139597f9..00000000 --- a/api-service/src/v2/connections/commandServiceConnection.ts +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/api-service/src/v2/controllers/DataOut/DataOutController.ts b/api-service/src/v2/controllers/DataOut/DataOutController.ts deleted file mode 100644 index a61bf545..00000000 --- a/api-service/src/v2/controllers/DataOut/DataOutController.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Request, Response } from "express"; -import logger from "../../logger"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { schemaValidation } from "../../services/ValidationService"; -import validationSchema from "./DataOutValidationSchema.json"; -import { validateQuery } from "./QueryValidator"; -import * as _ from "lodash"; -import { executeNativeQuery, executeSqlQuery } from "../../connections/druidConnection"; - -export const apiId = "api.data.out"; -const dataOut = async (req: Request, res: Response) => { - const datasetId = req.params?.datasetId; - const requestBody = req.body; - const msgid = _.get(req, "body.params.msgid"); - try { - const isValidSchema = schemaValidation(requestBody, validationSchema); - if (!isValidSchema?.isValid) { - logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); - } - const isValidQuery: any = await validateQuery(req.body, datasetId); - const query = _.get(req, "body.query", "") - - if (isValidQuery === true && _.isObject(query)) { - const result = await executeNativeQuery(query); - logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } - - if (isValidQuery === true && _.isString(query)) { - const result = await executeSqlQuery({ query }) - logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } - - else { - logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) - return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); - } - } - catch (err: any) { - logger.error({ ...err, apiId, code: err?.code || "INTERNAL_SERVER_ERROR" }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "INTERNAL_SERVER_ERROR", message: "Unable to process the query." } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -export default dataOut; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts deleted file mode 100644 index a264e138..00000000 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { Request, Response } from "express"; -import logger from "../../logger"; -import { generateDataSource, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; -import _ from "lodash"; -import DatasetCreate from "./DatasetCreateValidationSchema.json"; -import { schemaValidation } from "../../services/ValidationService"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import httpStatus from "http-status"; -import { defaultDatasetConfig, defaultMasterConfig } from "../../configs/DatasetConfigDefault"; -import { DatasetType } from "../../types/DatasetModels"; -import { query, sequelize } from "../../connections/databaseConnection"; -import { ErrorObject } from "../../types/ResponseModel"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; - -export const apiId = "api.datasets.create" - -const datasetCreate = async (req: Request, res: Response) => { - const requestBody = req.body - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - const transact = await sequelize.transaction() - try { - const datasetId = _.get(req, ["body", "request", "dataset_id"]) - setReqDatasetId(req, datasetId) - - const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) - - if (!isRequestValid.isValid) { - const code = "DATASET_INVALID_INPUT" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetBody = req.body.request; - const isDataSetExists = await checkDatasetExists(_.get(datasetBody, ["dataset_id"])); - if (isDataSetExists) { - const code = "DATASET_EXISTS" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Dataset Already exists with id:${_.get(datasetBody, "dataset_id")}` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset already exists", - statusCode: 409, - errCode: "CONFLICT" - } as ErrorObject, req, res); - } - - const duplicateDenormKeys = getDuplicateDenormKey(_.get(datasetBody, "denorm_config")) - if (!_.isEmpty(duplicateDenormKeys)) { - const code = "DATASET_DUPLICATE_DENORM_KEY" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) - return ResponseHandler.errorResponse({ - code, - statusCode: 400, - message: "Duplicate denorm key found", - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetPayload: any = await getDefaultValue(datasetBody); - const data = { ...datasetPayload, version_key: Date.now().toString() } - - const response = await DatasetDraft.create(data, { transaction: transact }) - - const { dataset_config, data_schema, id, dataset_id } = data - const datasourcePayload = generateDataSource({ indexCol: _.get(dataset_config, ["timestamp_key"]), data_schema, id, dataset_id }) - await DatasourceDraft.create(datasourcePayload, { transaction: transact }) - logger.info({ apiId, message: `Datasource created successsfully for the dataset:${id}` }) - - const transformationConfig: any = getTransformationConfig({ transformationPayload: _.get(datasetBody, "transformations_config"), datasetId: _.get(datasetPayload, "id") }) - if (!_.isEmpty(transformationConfig)) { - await DatasetTransformationsDraft.bulkCreate(transformationConfig), { transaction: transact }; - logger.info({ apiId, message: `Dataset transformations records created successsfully for dataset:${id}` }) - } - - await transact.commit() - const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: data.version_key } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - } catch (error: any) { - await transact.rollback() - const code = _.get(error, "code") || "DATASET_CREATION_FAILURE" - logger.error({ ...error, apiId, code, msgid, requestBody, resmsgid }) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to create dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const checkDatasetExists = async (dataset_id: string): Promise => { - const datasetExists = await getDraftDataset(dataset_id) - if (datasetExists) { - return true; - } else { - return false - } -} - -const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { - const { id, dataset_id, version = 1 } = requestPayload; - const recordId = !id && `${dataset_id}.${version}` - const modifyPayload = { ...requestPayload, ...(recordId && { id: recordId }) } - const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge(defaults, modifyPayload) - return datasetConfigs -} - -const getDatasetDefaults = async (payload: Record): Promise> => { - const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) - return datasetPayload -} - -const setRedisDBConfig = async (datasetConfig: Record): Promise> => { - let nextRedisDB = datasetConfig.redis_db; - const { results }: any = await query("SELECT nextval('redis_db_index')") - if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - return _.assign(datasetConfig, { "redis_db": nextRedisDB }) -} - -const getMasterDatasetDefaults = async (payload: Record): Promise> => { - const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload) - let datasetConfig = masterDatasetPayload.dataset_config - datasetConfig = await setRedisDBConfig(datasetConfig); - return _.assign(masterDatasetPayload, datasetConfig); -} - -const getDefaultHandler = (datasetType: string) => { - if (datasetType == DatasetType.Dataset) { - return getDatasetDefaults; - } else { - return getMasterDatasetDefaults; - } -} - -const getDefaultValue = async (payload: Record) => { - const datasetType = _.get(payload, "type"); - const getDatasetDefaults = getDefaultHandler(datasetType) - const datasetDefaults = await getDatasetDefaults(payload) - return _.omit(datasetDefaults, ["transformations_config"]) -} - -const getTransformationConfig = (configs: Record): Record => { - const { transformationPayload, datasetId } = configs - if (transformationPayload) { - - let transformations: any = [] - const transformationFieldKeys = _.flatten(_.map(transformationPayload, fields => _.get(fields, ["field_key"]))) - const duplicateFieldKeys: Array = getDuplicateConfigs(transformationFieldKeys) - - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ apiId, message: `Duplicate transformations provided by user are [${duplicateFieldKeys}]` }) - } - - _.forEach(transformationPayload, payload => { - const fieldKey = _.get(payload, "field_key") - const transformationExists = _.some(transformations, field => _.get(field, "field_key") == fieldKey) - if (!transformationExists) { - transformations = _.flatten(_.concat(transformations, { ...payload, id: `${datasetId}_${fieldKey}`, dataset_id: datasetId })) - } - }) - return transformations - } - return [] -} - -export default datasetCreate; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json deleted file mode 100644 index 9bf4af6f..00000000 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ /dev/null @@ -1,214 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string", - "enum": ["api.datasets.create"] - }, - "ver": { - "type": "string" - }, - "ts": { - "type": "string" - }, - "params": { - "type": "object", - "properties": { - "msgid": { - "type": "string" - } - }, - "required": ["msgid"], - "additionalProperties": false - }, - "request": { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "dataset_id": { - "type": "string", - "minLength": 1 - }, - "type": { - "type": "string", - "enum": ["dataset", "master-dataset"] - }, - "name": { - "type": "string", - "minLength": 1 - }, - "validation_config": { - "type": "object", - "properties": { - "validate": { - "type": "boolean" - }, - "mode": { - "type": "string", - "enum": ["Strict", "IgnoreNewFields"] - } - }, - "required": ["validate", "mode"], - "additionalProperties": false - }, - "extraction_config": { - "type": "object", - "properties": { - "is_batch_event": { - "type": "boolean" - }, - "extraction_key": { - "type": "string", - "minLength": 1 - }, - "dedup_config": { - "type": "object", - "properties": { - "drop_duplicates": { - "type": "boolean" - }, - "dedup_key": { - "type": "string", - "minLength": 1 - } - }, - "required": ["drop_duplicates", "dedup_key"], - "additionalProperties": false - } - }, - "required": ["is_batch_event", "extraction_key", "dedup_config"], - "additionalProperties": false - }, - "dedup_config": { - "type": "object", - "properties": { - "drop_duplicates": { - "type": "boolean" - }, - "dedup_key": { - "type": "string", - "minLength": 1 - } - }, - "required": ["drop_duplicates", "dedup_key"], - "additionalProperties": false - }, - "data_schema": { - "type": "object", - "properties": { - "$schema": { - "type": "string", - "minLength": 1 - }, - "type": { - "type": "string", - "minLength": 1 - }, - "properties": { - "type": "object", - "minProperties": 1 - }, - "additionalProperties": { - "type": "boolean", - "default": true - } - }, - "required": ["$schema", "type", "properties", "additionalProperties"], - "additionalProperties": false - }, - "denorm_config": { - "type": "object", - "properties": { - "denorm_fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "denorm_key": { - "type": "string", - "minLength": 1 - }, - "denorm_out_field": { - "type": "string", - "minLength": 1 - } - }, - "required": ["denorm_key", "denorm_out_field"], - "additionalProperties": false - } - } - }, - "required": ["denorm_fields"], - "additionalProperties": false - }, - "dataset_config": { - "type": "object", - "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string", - "minLength": 1 - }, - "file_upload_path": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - }, - "required": ["data_key", "timestamp_key"], - "additionalProperties": false - }, - "transformations_config": { - "type": "array", - "items": { - "type": "object", - "properties": { - "field_key": { - "type": "string", - "minLength": 1 - }, - "transformation_function": { - "type": "object" - }, - "mode": { - "type": "string", - "enum": [ - "Strict", - "Lenient" - ] - }, - "metadata": { - "type": "object" - } - }, - "additionalProperties": false, - "required": [ - "field_key", - "transformation_function", - "mode", - "metadata" - ] - } - }, - "tags": { - "type": "array", - "items": { - "type": "string", - "minLength":1 - } - } - }, - "required": ["dataset_id", "type", "name", "data_schema"], - "additionalProperties": false - } - }, - "required": ["id", "ver", "ts", "params", "request"], - "additionalProperties": false -} \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetList/DatasetList.ts b/api-service/src/v2/controllers/DatasetList/DatasetList.ts deleted file mode 100644 index 81d1a291..00000000 --- a/api-service/src/v2/controllers/DatasetList/DatasetList.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { Request, Response } from "express"; -import { schemaValidation } from "../../services/ValidationService"; -import DatasetCreate from "./DatasetListValidationSchema.json"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { ErrorObject } from "../../types/ResponseModel"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import logger from "../../logger"; -import _ from "lodash"; -import { Dataset } from "../../models/Dataset"; -import httpStatus from "http-status"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { DatasetTransformations } from "../../models/Transformation"; -import { DatasetStatus } from "../../types/DatasetModels"; - -export const apiId = "api.datasets.list" -export const errorCode = "DATASET_LIST_FAILURE" -const liveDatasetStatus = ["Live", "Retired"] -const draftDatasetStatus = ["Draft", "ReadyToPublish"] - -const datasetList = async (req: Request, res: Response) => { - const requestBody = req.body; - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - try { - const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) - if (!isRequestValid.isValid) { - const code = "DATASET_LIST_INPUT_INVALID" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetBody = req.body.request; - const datasetList = await getDatasetList(datasetBody) - const responseData = { data: datasetList, count: _.size(datasetList) } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - } catch (error: any) { - logger.error({ ...error, apiId, code: errorCode, msgid, requestBody, resmsgid }); - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: errorCode, message: "Failed to list dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const getDatasetList = async (request: Record): Promise> => { - const { filters = {}, sortBy = [] } = request || {}; - const datasets = await getAllDatasets(filters) - const sortedDatasets: any = getSortedDatasets(datasets, sortBy) - const datasetsList = await transformDatasetList(sortedDatasets) - return datasetsList; -} - -const getAllDatasets = async (filters: Record): Promise> => { - const datasetStatus = _.get(filters, "status"); - const { liveDatasetList, draftDatasetList } = await fetchDatasets({ datasetStatus, filters }) - return _.compact(_.concat(liveDatasetList, draftDatasetList)) -} - -const fetchDatasets = async (data: Record) => { - const { filters, datasetStatus } = data - if (_.isEmpty(datasetStatus)) { - const [liveDatasetList, draftDatasetList] = await Promise.all([getLiveDatasets(filters, liveDatasetStatus), getDraftDatasets(filters, draftDatasetStatus)]) - return { liveDatasetList, draftDatasetList } - } - let liveDatasetList, draftDatasetList; - const status = _.isArray(datasetStatus) ? datasetStatus : [datasetStatus] - const draftStatus = _.intersection(status, draftDatasetStatus); - const liveStatus = _.intersection(status, liveDatasetStatus); - if (_.size(liveStatus) > 0) { - liveDatasetList = await getLiveDatasets(filters, liveStatus) - } - if (_.size(draftStatus) > 0) { - draftDatasetList = await getDraftDatasets(filters, draftStatus) - } - return { liveDatasetList, draftDatasetList } -} - -const getSortedDatasets = (datasets: Record, sortOrder: Record): Record => { - if (!_.isEmpty(sortOrder)) { - const fieldValues = _.map(sortOrder, field => _.get(field, "field")) - const orderValues = _.map(sortOrder, field => _.get(field, "order")) - return _.orderBy(datasets, fieldValues, orderValues) - } - return datasets -} - -const transformDatasetList = async (datasets: Record) => { - const { liveDatasetId, draftDatasetId } = getDatasetId(datasets) - const [draftTransformations, liveTransformations] = await Promise.all([getDraftTransformations(draftDatasetId), getLiveTransformations(liveDatasetId)]) - const transformationList = _.concat(liveTransformations, draftTransformations) - const datasetList = _.map(datasets, dataset => { - const transformationConfig = _.compact(_.flatten(_.map(transformationList, (transformations: any) => { - const datasetId = _.get(dataset, "id") - const transformationId = _.get(transformations, "dataset_id") - if (datasetId === transformationId) { - return _.omit(transformations, ["dataset_id"]); - } - }))) - const liveDatasetVersion = _.get(dataset, "data_version") - const updatedList = liveDatasetVersion ? { ..._.omit(dataset, ["data_version"]), version: liveDatasetVersion } : dataset - return { ...updatedList, ...(!_.isEmpty(transformationConfig) && { "transformations_config": transformationConfig }) } - }) - return datasetList -} - -const getDatasetId = (datasets: Record) => { - const liveDatasets = _.filter(datasets, field => { - const { status } = field || {} - return status === DatasetStatus.Live || status === DatasetStatus.Retired - }) - const draftDatasets = _.filter(datasets, field => { - const { status } = field || {} - return status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish - }) - const liveDatasetId = _.map(liveDatasets, list => _.get(list, "id")) - const draftDatasetId = _.map(draftDatasets, list => _.get(list, "id")) - return { liveDatasetId, draftDatasetId } -} - -const getDraftDatasets = async (filters: Record, datasetStatus: Array): Promise> => { - return DatasetDraft.findAll({ where: { ...filters, ...(!_.isEmpty(datasetStatus) && { status: datasetStatus }) }, raw: true }); -} - -const getLiveDatasets = async (filters: Record, datasetStatus: Array): Promise> => { - return Dataset.findAll({ where: { ...filters, ...(!_.isEmpty(datasetStatus) && { status: datasetStatus }) }, raw: true }); -} - -const datasetTransformationAttributes = ["dataset_id", "field_key", "transformation_function", "mode", "metadata"] - -const getDraftTransformations = async (dataset_id: Array) => { - return DatasetTransformationsDraft.findAll({ where: { status: draftDatasetStatus, dataset_id }, attributes: datasetTransformationAttributes, raw: true }) -} - -const getLiveTransformations = async (dataset_id: Array) => { - return DatasetTransformations.findAll({ where: { status: liveDatasetStatus, dataset_id }, attributes: datasetTransformationAttributes, raw: true }) -} - -export default datasetList; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts deleted file mode 100644 index e6a586c2..00000000 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { Request, Response } from "express"; -import { DatasetStatus } from "../../types/DatasetModels"; -import _ from "lodash"; -import { validDatasetFields } from "../../configs/DatasetConfigDefault"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { ErrorObject } from "../../types/ResponseModel"; -import logger from "../../logger"; -import httpStatus from "http-status"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { DatasetTransformations } from "../../models/Transformation"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import { Dataset } from "../../models/Dataset"; -import { setReqDatasetId } from "../../services/DatasetService"; - -export const apiId = "api.datasets.read"; -export const errorCode = "DATASET_READ_FAILURE" - -const datasetRead = async (req: Request, res: Response) => { - const { dataset_id } = req.params; - const resmsgid = _.get(res, "resmsgid"); - try { - const { fields, status = DatasetStatus.Live } = req.query; - - setReqDatasetId(req, dataset_id) - - const invalidFields = !_.isEmpty(fields) ? getInvalidFields({ datasetFields: fields, status }) : [] - if (!_.isEmpty(invalidFields)) { - const code = "DATASET_INVALID_FIELDS" - logger.error({ code, apiId, dataset_id, resmsgid, message: `The specified fields [${invalidFields}] in the dataset cannot be found` }) - return ResponseHandler.errorResponse({ - code, - message: `The specified fields [${invalidFields}] in the dataset cannot be found.`, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetModel = getDatasetModel(status); - const fieldValue = !_.isEmpty(fields) ? transformFieldValues({ fields, status }) : "*" - let data: any = {} - if (!_.isEmpty(fieldValue)) { - const results = await datasetModel.findAll({ where: { id: dataset_id }, ...(fieldValue !== "*" && { attributes: fieldValue }), raw: true }) - if (_.isEmpty(results)) { - const code = "DATASET_NOT_FOUND" - logger.error({ code, apiId, dataset_id, resmsgid, message: `Dataset with the given dataset_id:${dataset_id} not found` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset with the given dataset_id not found", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - } - data = _.first(results) - } - - const responseData = await transformResponseData({ status, dataset_id, data, fields }); - logger.info({ apiId, resmsgid, message: `Dataset Read Successfully with id:${dataset_id}`, response: responseData }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - } catch (error: any) { - const code = _.get(error, "code") || errorCode - logger.error({ ...error, apiId, code, dataset_id, resmsgid }) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to read dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const getDatasetModel = (status: string | any) => { - if (status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) { - return DatasetDraft; - } - return Dataset; -} - -const getInvalidFields = (payload: Record): Record => { - const { datasetFields, status } = payload - const fieldValues = _.split(datasetFields, ",") - if (!(status == DatasetStatus.Draft || status == DatasetStatus.ReadyToPublish)) { - validDatasetFields.splice(_.indexOf(validDatasetFields, "version_key", 1)) - return _.difference(fieldValues, validDatasetFields) - } - const invalidFields = _.difference(fieldValues, validDatasetFields) - return invalidFields; -} - -const transformFieldValues = (datasetFields: Record) => { - const { status, fields } = datasetFields; - const updatedFields = _.remove(_.split(fields, ","), (newField) => newField !== "transformations_config") - if (!(status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) && _.includes(updatedFields, "version")) { - const fieldIndex = _.indexOf(updatedFields, "version") - updatedFields[fieldIndex] = "data_version" - } - return updatedFields -} - -const transformResponseData = async (payload: Record) => { - const { data, dataset_id, status, fields } = payload - const transformationConfigExist = _.includes(_.split(fields, ","), "transformations_config") - if (transformationConfigExist || _.isEmpty(fields)) { - const transformationModel = getTransfomationModel(status) - const transformations = await transformationModel.findAll({ where: { dataset_id }, raw: true, attributes: ["field_key", "transformation_function", "mode", "metadata"] }) - _.set(data, "transformations_config", transformations) - } - const liveDatasetVersion = _.get(data, "data_version") - const updatedResponse = liveDatasetVersion ? { ..._.omit(data, ["data_version"]), version: liveDatasetVersion } : data - return updatedResponse -} - -const getTransfomationModel = (status: string) => { - if (status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) { - return DatasetTransformationsDraft - } - return DatasetTransformations -} - -export default datasetRead; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts deleted file mode 100644 index 911aed8f..00000000 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ /dev/null @@ -1,408 +0,0 @@ -import httpStatus from "http-status"; -import { Request, Response } from "express"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { schemaValidation } from "../../services/ValidationService"; -import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; -import _ from "lodash"; -import { DatasetStatus } from "../../types/DatasetModels"; -import { ErrorObject } from "../../types/ResponseModel"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import logger from "../../logger"; -import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { generateDataSource, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService"; -import { sequelize } from "../../connections/databaseConnection"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; - -export const apiId = "api.datasets.update"; -export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" -export const errorCode = "DATASET_UPDATE_FAILURE" - -const datasetUpdate = async (req: Request, res: Response) => { - const requestBody = req.body - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - const transact = await sequelize.transaction(); - try { - const datasetId = _.get(req, ["body", "request", "dataset_id"]) - setReqDatasetId(req, datasetId) - - const isRequestValid: Record = schemaValidation(req.body, DatasetUpdate) - if (!isRequestValid.isValid) { - logger.error({ code: invalidInputErrCode, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code: invalidInputErrCode, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetBody = req.body.request - const { dataset_id, version_key, ...rest } = datasetBody - if (_.isEmpty(rest)) { - const code = "DATASET_UPDATE_NO_FIELDS" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) - return ResponseHandler.errorResponse({ - code, - message: "Provide atleast one field in addition to the dataset_id to update the dataset", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const { isDatasetExists, datasetStatus, invalidVersionKey, validVersionKey } = await checkDatasetExists(dataset_id, version_key); - if (!isDatasetExists) { - const code = "DATASET_NOT_EXISTS" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Dataset does not exists with id:${dataset_id}` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset does not exists to update", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - } - - if (isDatasetExists && datasetStatus != DatasetStatus.Draft) { - const code = "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Dataset with id:${dataset_id} cannot be updated as it is not in draft state` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset cannot be updated as it is not in draft state", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - if (invalidVersionKey) { - const code = "DATASET_OUTDATED" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `The dataset:${dataset_id} with version_key:${version_key} is outdated. Please try to fetch latest changes of the dataset with version key:${validVersionKey} and perform the updates` }) - return ResponseHandler.errorResponse({ - code, - message: "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", - statusCode: 409, - errCode: "CONFLICT" - } as ErrorObject, req, res); - } - - const duplicateDenormKeys = getDuplicateDenormKey(_.get(datasetBody, "denorm_config")) - if (!_.isEmpty(duplicateDenormKeys)) { - const code = "DATASET_DUPLICATE_DENORM_KEY" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) - return ResponseHandler.errorResponse({ - code, - statusCode: 400, - message: `Dataset contains duplicate denorm out keys:[${duplicateDenormKeys}]`, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const updatedDatasetConfigs = await getDatasetUpdatedConfigs(datasetBody); - const datasetPayload = await mergeExistingDataset(updatedDatasetConfigs) - - const transformationConfigs = _.get(datasetBody, "transformation_config") - await manageTransformations(transformationConfigs, dataset_id, transact); - - const { data_schema } = datasetBody - if (data_schema) { - const { dataset_config, id, dataset_id } = datasetPayload - const datasourcePayload = generateDataSource({ indexCol: _.get(dataset_config, ["timestamp_key"]), data_schema, id, dataset_id }) - await DatasourceDraft.update(datasourcePayload, { where: { id: _.get(datasourcePayload, "id") }, transaction: transact }) - } - - await DatasetDraft.update(datasetPayload, { where: { id: dataset_id }, transaction: transact }) - - await transact.commit(); - const responsedata = { message: "Dataset is updated successfully", id: dataset_id, version_key: _.get(datasetPayload, "version_key") } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Dataset updated successfully with id:${dataset_id}`, response: responsedata }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responsedata }); - } catch (error: any) { - await transact.rollback(); - const code = _.get(error, "code") || errorCode - logger.error({ ...error, apiId, code, msgid, requestBody, resmsgid }) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to update dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const manageTransformations = async (transformations: Record, datasetId: string, transact: any) => { - if (transformations) { - const transformationConfigs = await getTransformationConfigs(transformations, datasetId); - if (transformationConfigs) { - const { addTransformation, updateTransformation, deleteTransformation } = transformationConfigs; - - if (!_.isEmpty(addTransformation)) { - await DatasetTransformationsDraft.bulkCreate(addTransformation, { transaction: transact }); - } - - if (!_.isEmpty(updateTransformation)) { - await Promise.all(updateTransformation.map((record: any) => DatasetTransformationsDraft.update(record, { where: { id: record.id }, transaction: transact }))); - } - - if (!_.isEmpty(deleteTransformation)) { - await DatasetTransformationsDraft.destroy({ where: { id: deleteTransformation }, transaction: transact }); - } - } - } -} - -const checkDatasetExists = async (dataset_id: string, version_key: string): Promise> => { - const datasetExists: Record | null = await getExistingDataset(dataset_id) - if (datasetExists) { - const validVersionKey = _.get(datasetExists, "version_key") - if (validVersionKey !== version_key) { - return { isDatasetExists: true, datasetStatus: datasetExists.status, invalidVersionKey: true, validVersionKey } - } - return { isDatasetExists: true, datasetStatus: datasetExists.status } - } else { - return { isDatasetExists: false, datasetStatus: "" } - } -} - -const getDatasetUpdatedConfigs = async (payload: Record): Promise> => { - const { validation_config, extraction_config, dedup_config, tags, denorm_config, dataset_id } = payload - const existingDataset: any = await getExistingDataset(dataset_id) - const updatedConfigs = { - validation_config: validation_config ? setValidationConfigs(validation_config) : null, - extraction_config: extraction_config ? setExtractionConfigs(extraction_config) : null, - dedup_config: dedup_config ? setDedupConfigs(dedup_config) : null, - tags: tags ? getUpdatedTags(tags, _.get(existingDataset, "tags")) : null, - denorm_config: denorm_config ? setDenormConfigs(denorm_config, _.get(existingDataset, ["denorm_config"])) : null, - version_key: Date.now().toString() - } - return _.pickBy({ ...payload, ...updatedConfigs }, _.identity) -} - -const getDuplicateDenormKey = (denormConfig: Record): Array => { - if (denormConfig && _.isArray(_.get(denormConfig, "denorm_fields"))) { - const denormFields = _.get(denormConfig, "denorm_fields") - const denormOutKeys = _.compact(_.map(denormFields, field => _.get(field, "action") == "add" && _.get(field, "values.denorm_out_field"))) - const duplicateDenormKeys: Array = _.filter(denormOutKeys, (item: string, index: number) => _.indexOf(denormOutKeys, item) !== index); - return duplicateDenormKeys; - } - return [] -} - -const setValidationConfigs = (configs: Record): Record => { - const validationStatus = _.get(configs, "validate") - if (!validationStatus) { - _.set(configs, "mode", defaultDatasetConfig.validation_config.mode) - } - return configs; -} - -const setExtractionConfigs = (configs: Record): Record => { - const isBatchEvent = _.get(configs, "is_batch_event") - if (!isBatchEvent) { - return _.merge(configs, defaultDatasetConfig.extraction_config) - } - return configs -} - -const setDedupConfigs = (configs: Record): Record => { - const dropDuplicates = _.get(configs, "drop_duplicates") - if (!dropDuplicates) { - return { ...defaultDatasetConfig.dedup_config, drop_duplicates: dropDuplicates } - } - return configs -} - -const getUpdatedTags = (newTagsPayload: Record, datasetTags: Array): Record => { - - const getTagsPayload = (action: string) => { - return _.compact(_.flatten(_.map(newTagsPayload, fields => { - if (fields.action == action) return _.map(fields.values, value => _.toLower(value)) - }))) - } - const tagsToAdd = getTagsPayload("add") - const tagsToRemove = getTagsPayload("remove") - - const duplicateTagsToAdd: Array = getDuplicateConfigs(tagsToAdd) - if (!_.isEmpty(duplicateTagsToAdd)) { - logger.info({ apiId, message: `Duplicate tags provided by user to add are [${duplicateTagsToAdd}]` }) - } - - const duplicateTagsToRemove: Array = getDuplicateConfigs(tagsToRemove) - if (!_.isEmpty(duplicateTagsToRemove)) { - logger.info({ apiId, message: `Duplicate tags provided by user to remove are [${duplicateTagsToRemove}]` }) - } - - const checkTagToAdd = _.intersection(datasetTags, _.uniq(tagsToAdd)) - if (_.size(checkTagToAdd)) { - throw { - code: "DATASET_TAGS_EXISTS", - message: "Dataset tags already exist", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } - - const checkTagToRemove = _.intersection(datasetTags, _.uniq(tagsToRemove)) - if (_.size(checkTagToRemove) !== _.size(_.uniq(tagsToRemove))) { - throw { - code: "DATASET_TAGS_DO_NOT_EXIST", - message: "Dataset tags do not exist to remove", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - const tagsAfterAddition = _.concat(datasetTags, tagsToAdd) - const updatedTags = _.difference(tagsAfterAddition, tagsToRemove) - return _.flatten(updatedTags) -} - -const getTransformationConfigs = async (newTransformationPayload: Record, dataset_id: string): Promise> => { - const datasetTransformations: any = await getDraftTransformations(dataset_id) - let addTransformation: Record = [] - let updateTransformation: Record = [] - let deleteTransformation: Array = [] - - const existingTransformationsFields = _.map(datasetTransformations, config => config?.field_key) - const transformationFieldKeys = _.flatten(_.map(newTransformationPayload, fields => { - if (fields.action == "add") return fields.values.field_key - })) - - const duplicateFieldKeys: Array = getDuplicateConfigs(transformationFieldKeys) - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ apiId, message: `Duplicate transformations provided by user are [${duplicateFieldKeys}]` }) - } - - const getTransformationPayload = (action: string) => { - return _.compact(_.flatten(_.map(newTransformationPayload, payload => { - if (payload.action == action) return payload.values - }))) - } - const transformationsToAdd = getTransformationPayload("add") - const transformationsToUpdate = getTransformationPayload("update") - const transformationsToRemove = getTransformationPayload("remove") - - const checkTransformations = (transformationPayload: Record) => { - return _.intersection(existingTransformationsFields, _.map(transformationPayload, payload => _.get(payload, ["field_key"]))) - } - const checkTransformationToAdd = checkTransformations(transformationsToAdd) - const checkTransformationToRemove = checkTransformations(transformationsToRemove) - const checkTransformationToUpdate = checkTransformations(transformationsToUpdate) - - if (_.size(checkTransformationToAdd)) { - throw { - code: "DATASET_TRANSFORMATIONS_EXIST", - message: "Dataset transformations already exists", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } - - const transformationExistsToUpdate = _.every(transformationsToUpdate, payload => _.includes(checkTransformationToUpdate, payload.field_key)) - if (!transformationExistsToUpdate) { - throw { - code: "DATASET_TRANSFORMATIONS_DO_NOT_EXIST", - message: "Dataset transformations do not exist to update", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - const isTransformationExistsToRemove = _.every(transformationsToRemove, payload => _.includes(checkTransformationToRemove, payload.field_key)) - if (!isTransformationExistsToRemove) { - throw { - code: "DATASET_TRANSFORMATIONS_DO_NOT_EXIST", - message: "Dataset transformations do not exist to remove", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - _.forEach(newTransformationPayload, fields => { - const { values, action } = fields - const fieldKey = _.get(values, "field_key") - if (action == "remove") { - const deleteFieldKey: string = `${dataset_id}_${fieldKey}` - deleteTransformation = _.uniq(_.flatten(_.concat(deleteTransformation, deleteFieldKey))) - } - if (action == "add") { - const transformationExists = _.some(addTransformation, field => field.field_key == fieldKey) - if (!transformationExists) { - addTransformation = _.flatten(_.concat(addTransformation, { ...values, id: `${dataset_id}_${fieldKey}`, dataset_id })) - } - } - if (action == "update") { - const existingTransformations = _.filter(datasetTransformations, transformationField => transformationField.field_key == fieldKey) - const updatedTransformations = _.merge(_.get(existingTransformations, "[0]"), values) - updateTransformation = _.flatten(_.concat(updateTransformation, updatedTransformations)) - } - }) - return { addTransformation, deleteTransformation, updateTransformation } -} - -const setDenormConfigs = (newDenormPayload: Record, datasetDenormConfigs: Record): Record => { - const datasetDenormFieldsKeys = _.map(_.get(datasetDenormConfigs, "denorm_fields"), fields => fields?.denorm_out_field) - const { denorm_fields } = newDenormPayload; - const existingDenormFields = _.get(datasetDenormConfigs, "denorm_fields") || [] - - const getDenormPayload = (action: string) => { - return _.compact(_.flatten(_.map(denorm_fields, payload => { - if (payload.action == action) return payload.values - }))) - } - const denormsToAdd = getDenormPayload("add") - const denormsToRemove = getDenormPayload("remove") - - const denormFieldsToRemove = _.map(denormsToRemove, field => field.denorm_out_field) - const duplicateFieldKeys: Array = getDuplicateConfigs(denormFieldsToRemove) - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ apiId, message: `Duplicate denorm out fields provided by user are [${duplicateFieldKeys}]` }) - } - - const checkDenormKeys = (denormKeys: Record) => { - return _.intersection(datasetDenormFieldsKeys, _.map(denormKeys, payload => _.get(payload, ["denorm_out_field"]))) - } - const checkDenormsToAdd = checkDenormKeys(denormsToAdd) - const checkDenormsToRemove = checkDenormKeys(denormsToRemove) - - if (_.size(checkDenormsToAdd)) { - throw { - code: "DATASET_DENORM_EXISTS", - message: "Denorm fields already exist", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } - - const isDenormExists = _.every(denormsToRemove, payload => _.includes(checkDenormsToRemove, payload.denorm_out_field)) - if (!isDenormExists) { - throw { - code: "DATASET_DENORM_DO_NOT_EXIST", - message: "Denorm fields do not exist to remove", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - const denormsFields = _.concat(existingDenormFields, denormsToAdd) - const updatedDenormFields = _.filter(denormsFields, fields => !_.includes(checkDenormsToRemove, fields.denorm_out_field)) - return { ...datasetDenormConfigs, denorm_fields: updatedDenormFields } -} - -const mergeExistingDataset = async (configs: Record): Promise> => { - const existingDataset = await getExistingDataset(_.get(configs, "dataset_id")) - const mergedData = _.mergeWith(existingDataset, configs, (existingValue, newValue) => { - if (_.isArray(existingValue) && _.isEmpty(newValue)) { - return []; - } - if (_.isArray(existingValue) && !_.isEmpty(newValue)) { - return newValue - } - }); - return _.omit(mergedData, ["dataset_id", "transformation_config", "created_date"]) -} - -export const getExistingDataset = async (id: string) => { - return DatasetDraft.findOne({ where: { id }, raw: true }) -} - -export default datasetUpdate; diff --git a/api-service/src/v2/logger/index.ts b/api-service/src/v2/logger/index.ts deleted file mode 100644 index d43a8625..00000000 --- a/api-service/src/v2/logger/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -import winston from "winston"; - - winston.configure({ - level: "info", - transports: [new winston.transports.Console()], -}); - -const innerLogger = new (winston.Logger)(); -const logger = { - error: (...data: any) => innerLogger.error("", data), - info: (...data: any) => innerLogger.info("", data), - warn: (...data: any) => innerLogger.warn("", data), -} - -export default logger; \ No newline at end of file diff --git a/api-service/src/v2/routes/Router.ts b/api-service/src/v2/routes/Router.ts deleted file mode 100644 index a34114ee..00000000 --- a/api-service/src/v2/routes/Router.ts +++ /dev/null @@ -1,43 +0,0 @@ -import express from "express"; -import dataIn from "../controllers/DataIngestion/DataIngestionController"; -import DatasetCreate from "../controllers/DatasetCreate/DatasetCreate"; -import dataOut from "../controllers/DataOut/DataOutController"; -import DatasetUpdate from "../controllers/DatasetUpdate/DatasetUpdate"; -import DatasetRead from "../controllers/DatasetRead/DatasetRead"; -import DatasetList from "../controllers/DatasetList/DatasetList" -import { dataExhaust } from "../controllers/DataExhaust/DataExhaustController"; -import { onRequest } from "../metrics/prometheus/helpers"; - -import { Entity } from "../types/MetricModel"; -import { createQueryTemplate } from "../controllers/CreateQueryTemplate/CreateTemplateController"; -import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; -import { readQueryTemplate } from "../controllers/ReadQueryTemplate/ReadTemplateController"; -import { deleteQueryTemplate } from "../controllers/DeleteQueryTemplate/DeleteTemplateController"; -import { listQueryTemplates } from "../controllers/ListQueryTemplates/ListTemplatesController"; -import { queryTemplate } from "../controllers/QueryTemplate/QueryTemplateController"; -import { updateQueryTemplate } from "../controllers/UpdateQueryTemplate/UpdateTemplateController"; -import { eventValidation } from "../controllers/EventValidation/EventValidation"; -import GenerateSignedURL from "../controllers/GenerateSignedURL/GenerateSignedURL"; -import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; - -export const router = express.Router(); - -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); - -//Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); - diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts deleted file mode 100644 index feb340f3..00000000 --- a/api-service/src/v2/services/DatasetService.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { Dataset } from "../models/Dataset"; -import _ from "lodash"; -import { DatasetDraft } from "../models/DatasetDraft"; -import { DatasetTransformationsDraft } from "../models/TransformationDraft"; -import { Request } from "express"; -import { generateIngestionSpec } from "./IngestionService"; -import { ingestionConfig } from "../configs/IngestionConfig"; -import { DatasetTransformations } from "../models/Transformation"; - -export const getDataset = async (datasetId: string, raw = false): Promise => { - const dataset = await Dataset.findOne({ - where: { - id: datasetId, - }, - raw: raw - }); - return dataset -} - -export const getDuplicateDenormKey = (denormConfig: Record): Array => { - if (denormConfig && _.isArray(_.get(denormConfig, "denorm_fields"))) { - const denormFields = _.get(denormConfig, "denorm_fields") - const denormOutKeys = _.map(denormFields, field => _.get(field, "denorm_out_field")) - const duplicateDenormKeys: Array = _.filter(denormOutKeys, (item: string, index: number) => _.indexOf(denormOutKeys, item) !== index); - return duplicateDenormKeys; - } - return [] -} - -export const getDraftDataset = async (dataset_id: string) => { - return DatasetDraft.findOne({ where: { dataset_id }, raw: true }); -} - -export const getDraftTransformations = async (dataset_id: string) => { - return DatasetTransformationsDraft.findAll({ where: { dataset_id }, raw: true }); -} - -export const getTransformations = async (dataset_id: string) => { - return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); -} - -export const setReqDatasetId = (req: Request, dataset_id: string) => { - if (dataset_id) { - return _.set(req, "dataset_id", dataset_id) - } -} - -export const getDuplicateConfigs = (configs: Array) => { - return _.filter(configs, (item: string, index: number) => _.indexOf(configs, item) !== index); -} - -export const generateDataSource = (payload: Record) => { - const { id } = payload - const ingestionSpec = generateIngestionSpec(payload) - const dataSource = getDataSource({ ingestionSpec, id }) - return dataSource -} - -const getDataSource = (ingestionPayload: Record) => { - const { ingestionSpec, id } = ingestionPayload - const dataSource = `${id}_${_.toLower(ingestionConfig.granularitySpec.segmentGranularity)}` - const dataSourceId = `${id}_${dataSource}` - return { - id: dataSourceId, - datasource: dataSource, - dataset_id: id, - ingestion_spec: ingestionSpec, - datasource_ref: dataSource - } -} \ No newline at end of file diff --git a/api-service/src/v2/services/DatasourceService.ts b/api-service/src/v2/services/DatasourceService.ts deleted file mode 100644 index 687fc860..00000000 --- a/api-service/src/v2/services/DatasourceService.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Datasource } from "../models/Datasource"; -import { DatasourceDraft } from "../models/DatasourceDraft"; - -export const getDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await Datasource.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - -export const getDraftDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await DatasourceDraft.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - -export const getDatasource = async (datasetId: string) => { - const dataSource = await Datasource.findOne({ - where: { - dataset_id: datasetId, - }, - }); - return dataSource -} \ No newline at end of file diff --git a/api-service/src/v2/services/HealthService.ts b/api-service/src/v2/services/HealthService.ts deleted file mode 100644 index c8eca736..00000000 --- a/api-service/src/v2/services/HealthService.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { AxiosInstance } from "axios"; -import { NextFunction, Request, Response } from "express"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { IConnector } from "../types/DatasetModels"; -// import { DbConnector } from "../connections/databaseConnection"; -// import { KafkaConnector } from "../connections/kafkaConnection"; - - - - -export class HealthService { - // private dbConnector: DbConnector; - // private kafkaConnector: KafkaConnector; - private httpDruidConnector: AxiosInstance; - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: any, kafkaConnector:any, httpDruidConnector: IConnector) { - this.errorHandler = new ErrorResponseHandler("HealthService"); - this.httpDruidConnector = httpDruidConnector.connect() - // this.dbConnector = dbConnector; - // this.kafkaConnector = kafkaConnector; - } - - checkHealth(req: Request, res: Response, next: NextFunction) { - Promise.all([this.checkDruidHealth(), this.checkKafkaHealth(), this.checkPostgresHealth()]) - .then(() => { - ResponseHandler.successResponse(req, res, { status: 200, data: {} }) - }).catch(error => { - this.errorHandler.handleError(req, res, next, error) - }) - } - - private async checkDruidHealth() { - await this.httpDruidConnector.get("/status/health") - } - - private async checkKafkaHealth() { - // await this.kafkaConnector.connect() - } - - private async checkPostgresHealth() { - // await this.dbConnector.health() - } - -} diff --git a/api-service/src/v2/services/IngestionService.ts b/api-service/src/v2/services/IngestionService.ts deleted file mode 100644 index 9ab64de6..00000000 --- a/api-service/src/v2/services/IngestionService.ts +++ /dev/null @@ -1,260 +0,0 @@ -import _ from "lodash"; -import { ingestionConfig } from "../configs/IngestionConfig"; -import { config } from "../configs/Config"; -import { ErrorObject } from "../types/ResponseModel"; -import logger from "../logger"; -import { IngestionSpecModel, IngestionSpecObject } from "../types/IngestionModels"; -const defaultIndexCol = ingestionConfig.indexCol["Event Arrival Time"] - -const connectorSpecObj = { - "flattenSpec": { - "type": "path", - "expr": "$.obsrv_meta.source.['connector']", - "name": "obsrv.meta.source.connector" - }, - "dimensions": { - "type": "string", - "name": "obsrv.meta.source.connector" - }, - "fieldType": "dimensions" -} - -const connectorInstanceSpecObj = { - "flattenSpec": { - "type": "path", - "expr": "$.obsrv_meta.source.['connectorInstance']", - "name": "obsrv.meta.source.id" - }, - "dimensions": { - "type": "string", - "name": "obsrv.meta.source.id" - }, - "fieldType": "dimensions" -} - -export const generateIngestionSpec = (payload: Record) => { - const { indexCol = defaultIndexCol, data_schema, id, dataset_id } = payload - const isValidTimestamp = checkTimestampCol({ indexCol, data_schema }) - if (!isValidTimestamp) { - throw { - "code": "DATASET_TIMESTAMP_NOT_FOUND", - "message": "Provided timestamp key not found in the data schema", - "statusCode": 400, - "errCode": "BAD_REQUEST" - } as ErrorObject - } - const simplifiedSpec = generateExpression(_.get(data_schema, "properties"), indexCol); - const generatedSpec = process(simplifiedSpec, indexCol) - const ingestionTemplate = generateIngestionTemplate({ generatedSpec, id, indexCol, dataset_id, type: "druid" }) - return ingestionTemplate -} - -const generateIngestionTemplate = (payload: Record) => { - const { type, ...rest } = payload - switch (type) { - case "druid": - return getDruidIngestionTemplate(rest); - default: - return null; - } -} - -const checkTimestampCol = (schema: Record) => { - const { indexCol, data_schema } = schema - if (indexCol !== defaultIndexCol) { - const properties = generateFlattenSchema(data_schema); - const exists = _.find(properties, (value) => `$.${indexCol}` === value.path); - return exists - } - return true -} - -const process = (spec: Map, indexCol: string): IngestionSpecModel => { - const colValues = Array.from(spec.values()) - const dimensions = filterDimensionCols(colValues) - return { - "dimensions": getObjByKey(dimensions, "dimensions"), - "metrics": filterMetricsCols(spec), - "flattenSpec": filterFlattenSpec(colValues, indexCol) - } -} - -const filterFlattenSpec = (column: Record, indexCol: string) => { - let flattenSpec = getObjByKey(column, "flattenSpec") - if (indexCol === defaultIndexCol) { - const indexColDefaultSpec = { - "expr": ingestionConfig.syncts_path, - "name": ingestionConfig.indexCol["Event Arrival Time"], - "type": "path" - } - flattenSpec = _.concat(flattenSpec, indexColDefaultSpec) - } - return flattenSpec -} - -const filterMetricsCols = (spec: Record) => { - const metrics = _.filter(spec, cols => cols.fieldType === "metrics") - const metricCols = _.map(metrics, (value) => value["dimensions"]) - const updatedMetrics: any[] = [] - _.map(metricCols, (value) => { - value["fieldName"] = value["fieldName"] || value["name"]; - updatedMetrics.push(value); - }); - return updatedMetrics; -} - -const filterDimensionCols = (spec: Record) => { - const dimensionCols = _.filter(spec, cols => cols.fieldType === "dimensions") - return dimensionCols -} - -const getObjByKey = (sample: any, key: string) => { - const result = _.map(sample, (value) => _.get(value, key)); - return _.compact(result) -} - -export const generateFlattenSchema = (sample: Map) => { - const array: any[] = []; - const flattenValues = (data: any, path: string) => { - _.map(data, (value, key) => { - if (_.isPlainObject(value) && _.has(value, "properties")) { - array.push(flattenSchema(key, `${path}.${key}`)) - flattenValues(value["properties"], `${path}.${key}`); - } else if (_.isPlainObject(value)) { - if (value.type === "array") { - array.push(flattenSchema(key, `${path}.${key}`)) - if (_.has(value, "items") && _.has(value["items"], "properties")) { - flattenValues(value["items"]["properties"], `${path}.${key}`); - } - } else { - array.push(flattenSchema(key, `${path}.${key}`)) - } - } - }) - } - const properties = _.get(sample, "properties") - flattenValues(properties, "$") - return array -} - -const flattenSchema = (expr: string, path: string) => { - return { "property": expr, "path": path } -} - -export const generateExpression = (sample: Map, indexCol: string): Map => { - const flattendedSchema = new Map(); - const flattenExpression = (data: any, path: string) => { - _.map(data, (value, key) => { - if (_.isPlainObject(value) && (_.has(value, "properties"))) { - flattenExpression(value["properties"], `${path}.${key}`); - } else if (_.isPlainObject(value)) { - if (value.type === "array") { - if (_.has(value, "items") && _.has(value["items"], "properties")) { - flattenExpression(value["items"]["properties"], `${path}.${key}[*]`); - } else { - const objectType = getObjectType(value.type) - const specObject = createSpecObj({ expression: `${path}.['${key}'][*]`, objectType, name: `${path}.${key}`, indexCol }) - flattendedSchema.set(`${path}.${key}`, specObject) - } - } else if (value.type == "object" && (!_.has(value, "properties"))) { - const objectType = getObjectType(value.type) - const specObject = createSpecObj({ expression: `${path}.['${key}']`, objectType, name: `${path}.${key}`, indexCol }) - flattendedSchema.set(`${path}.${key}`, specObject) - logger.warn(`Found empty object without properties in the schema with Key: ${key}, Object: ${JSON.stringify(value)}`) - } - else { - const objectType = getObjectType(value.type) - const specObject = createSpecObj({ expression: `${path}.['${key}']`, objectType, name: `${path}.${key}`, indexCol }) - flattendedSchema.set(`${path}.${key}`, specObject) - } - } - }) - } - flattenExpression(sample, "$") - flattendedSchema.set("obsrv.meta.source.connector", connectorSpecObj).set("obsrv.meta.source.connector.instance", connectorInstanceSpecObj) - return flattendedSchema -} - -const createSpecObj = (payload: Record): IngestionSpecObject => { - const { expression, objectType, name, indexCol } = payload - const propertyName = _.replace(name.replace("[*]", ""), "$.", "") - const specObj = { - "flattenSpec": { - "type": "path", - "expr": expression, - "name": propertyName - }, - "dimensions": { - "type": objectType, - "name": propertyName - }, - "fieldType": "dimensions" - } - if ([indexCol].includes(specObj.flattenSpec.name)) { - specObj.fieldType = "timestamp" - } - return specObj -} - -const getObjectType = (type: string): string => { - switch (type) { - case "number": return "double"; - case "integer": return "long"; - case "object": return "json"; - case "boolean": return "string"; - default: return type; - } -} - -export const getDruidIngestionTemplate = (payload: Record) => { - const { id, generatedSpec, indexCol, dataset_id } = payload - const { dimensions, metrics, flattenSpec } = generatedSpec - const dataSource = `${id}_${_.toLower(ingestionConfig.granularitySpec.segmentGranularity)}` - return { - "type": "kafka", - "spec": { - "dataSchema": { - "dataSource": dataSource, - "dimensionsSpec": { "dimensions": dimensions }, - "timestampSpec": { "column": indexCol, "format": "auto" }, - "metricsSpec": metrics, - "granularitySpec": getGranularityObj(), - }, - "tuningConfig": { - "type": "kafka", - "maxBytesInMemory": ingestionConfig.maxBytesInMemory, - "maxRowsPerSegment": ingestionConfig.tuningConfig.maxRowPerSegment, - "logParseExceptions": true - }, - "ioConfig": getIOConfigObj(flattenSpec, dataset_id) - } - } -} - -const getGranularityObj = () => { - return { - "type": "uniform", - "segmentGranularity": ingestionConfig.granularitySpec.segmentGranularity, - "queryGranularity": ingestionConfig.query_granularity, - "rollup": ingestionConfig.granularitySpec.rollup - } -} - -const getIOConfigObj = (flattenSpec: Record, topic: string): Record => { - return { - "type": "kafka", - "topic": topic, - "consumerProperties": { "bootstrap.servers": config.telemetry_service_config.kafka.config.brokers[0] }, - "taskCount": ingestionConfig.tuningConfig.taskCount, - "replicas": 1, - "taskDuration": ingestionConfig.ioconfig.taskDuration, - "useEarliestOffset": ingestionConfig.use_earliest_offset, - "completionTimeout": ingestionConfig.completion_timeout, - "inputFormat": { - "type": "json", "flattenSpec": { - "useFieldDiscovery": true, "fields": flattenSpec - } - }, - "appendToExisting": false - } -} \ No newline at end of file diff --git a/api-service/src/v2/telemetry/telemetryActions.ts b/api-service/src/v2/telemetry/telemetryActions.ts deleted file mode 100644 index bbee6faa..00000000 --- a/api-service/src/v2/telemetry/telemetryActions.ts +++ /dev/null @@ -1,21 +0,0 @@ -export default { - "createDataset": "dataset:create", - "updateDataset": "dataset:update", - "readDataset": "dataset:read", - "listDatasets": "dataset:list", - "createDatasource": "datasource:create", - "updateDatasource": "datasource:update", - "getDatasource": "datasource:get", - "listDatasource": "datasource:list", - "createTransformation": "transformation:create", - "updateTransformation": "transformation:update", - "createDatasetSourceConfig": "datasetSourceConfig:create", - "updateDatasetSourceConfig": "datasetSourceConfig:update", - "getatasetSourceConfig": "datasetSourceConfig:get", - "listDatasetSourceConfig": "datasetSourceConfig:list", - "nativeQuery": "dataset:query:native", - "sqlQuery": "dataset:query:sql", - "ingestEvents": "dataset:events:ingest", - "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" -} \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DataOutTest/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DataOutTest/Fixtures.ts deleted file mode 100644 index 8f9bd9f7..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DataOutTest/Fixtures.ts +++ /dev/null @@ -1,23 +0,0 @@ -export const TestQueries = { - VALID_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"test","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2024-01-31/2024-02-01"]},"granularity":"week","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}]}}', - HIGH_LIMIT_NATIVE_QUERY: '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2024-11-31/2024-12-01"]},"granularity":"day","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}],"limit":10000,"threshold":10000}}', - WITHOUT_THRESOLD_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events"},"query":{"queryType":"timeBoundary","dimension":"content_status","metric":"count","granularity":"all","intervals":["2020-12-21/2020-12-22"],"aggregations":[]}}', - VALID_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"test","aggregationLevel":"week"},"query":"SELECT * FROM \\"test\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}', - HIGH_LIMIT_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events"},"querySql":{"query":"SELECT msgid FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-22\' LIMIT 100000"}}', - HIGH_DATE_RANGE_SQL_QUERY: - `{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2022-02-12' LIMIT 10"}`, - LIMIT_IS_NAN: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"dataSource":"telemetry-events","aggregationLevel":"week"},"query":"SELECT content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-12\' LIMIT 100"}', - DATASOURCE_NOT_FOUND: '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry"},"query":"SELECT content_status FROM \\"telemetry\\" LIMIT 5"}', - INVALID_DATE_RANGE_NATIVE: '{"id":"api.data.out","ver":"1.0","ts":"1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","table":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2023-01-31/2023-04-01"]},"granularity":"day","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"}]}}', - INVALID_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry","aggregationLevel":"week"},"query":"SELECT * FROM \\"telemetry\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}', - VALID_SQL_QUERY_WITHOUT_LIMIT: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry","aggregationLevel":"week"},"query":"SELECT * FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\'"}', - VALID_INTERVAL: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":"2024-01-31/2024-02-01","granularity":"week","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}]}}', -} \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts deleted file mode 100644 index 67344e5a..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts +++ /dev/null @@ -1,184 +0,0 @@ -import app from "../../../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetList/DatasetList"; -import { TestInputsForDatasetList } from "./Fixtures"; -import { Dataset } from "../../../models/Dataset"; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - -describe("DATASET LIST API", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset list success: When no filters are provided", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITHOUT_FILTERS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.count.should.be.eq(2) - res.body.params.msgid.should.be.eq(msgid) - const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }, { ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }]) - result.should.be.eq(expectedResult) - done(); - }); - }); - - it("Dataset list success: When status filter provided in request payload", (done) => { - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITH_STATUS_FILTERS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.count.should.be.eq(1) - res.body.params.msgid.should.be.eq(msgid) - const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }]) - result.should.be.eq(expectedResult) - done(); - }); - }); - - it("Dataset list success: When type filter provided in request payload", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITH_TYPE_FILTERS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.count.should.be.eq(1) - res.body.params.msgid.should.be.eq(msgid) - const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }]) - result.should.be.eq(expectedResult) - done(); - }); - }); - - it("Dataset list success: When sortBy is provided in request payload", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITH_SORTBY) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.count.should.be.eq(2) - res.body.params.msgid.should.be.eq(msgid) - const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }, { ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }]) - result.should.be.eq(expectedResult) - done(); - }); - }); - - it("Dataset list failure: Invalid request payload provided", (done) => { - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.INVALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq("DATASET_LIST_INPUT_INVALID") - expect(res.body.error.message).to.match(/^(.+) must be equal to one of the allowed values$/) - done(); - }); - }); - - - it("Dataset list failure: Connection to the database failed", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.reject() - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITHOUT_FILTERS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to list dataset") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts deleted file mode 100644 index a905b1c0..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts +++ /dev/null @@ -1,234 +0,0 @@ -export const TestInputsForDatasetList = { - VALID_DRAFT_DATASET_SCHEMA: { - "dataset_id": "telemetry", - "id": "telemetry.1", - "name": "telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "msgid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, - "tags": [ - "tag1", - "tag2" - ], - "status": "Draft", - "version": 1, - "client_state": {}, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "2024-04-15 07:51:49.49", - "update_date": "", - "published_date": "" - }, - VALID_LIVE_DATASET_SCHEMA: { - "dataset_id": "sb-telemetry", - "id": "sb-telemetry", - "name": "sb-telemetry", - "type": "master-dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "msgid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, - "tags": [ - "tag1", - "tag2" - ], - "status": "Live", - "data_version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "2024-04-16 07:51:49.49", - "update_date": "", - "published_date": "" - }, - TRANSFORMATIONS_DRAFT_SCHEMA: { - "dataset_id": "telemetry.1", - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - }, - TRANSFORMATIONS_LIVE_SCHEMA: { - "dataset_id": "sb-telemetry", - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - } - }, - - REQUEST_WITHOUT_FILTERS: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": {} - }, - REQUEST_WITH_STATUS_FILTERS: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "filters": { status: ["Draft"] } - } - }, - REQUEST_WITH_TYPE_FILTERS: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "filters": { status: "Live", type: "master-dataset" } - } - }, - REQUEST_WITH_SORTBY: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "sortBy": [{ "field": "created_date", "order": "asc" }] - } - }, - INVALID_REQUEST: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "filters": { status: ["Ready"] } - } - } -} \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts deleted file mode 100644 index 48735d29..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ /dev/null @@ -1,157 +0,0 @@ -import app from "../../../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; -import { TestInputsForDatasetRead } from "./Fixtures"; -import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { Dataset } from "../../../models/Dataset"; -import { DatasetDraft } from "../../../models/DatasetDraft"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("DATASET READ API", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset read success: When minimal fields requested", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ 'name': 'sb-telemetry', 'data_version': 1 }]) - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=name,version") - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') - const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ 'name': 'sb-telemetry', 'version': 1 })) - done(); - }); - }); - - it("Dataset read success: Fetch all dataset fields when fields param is empty", (done) => { - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DRAFT_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft") - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.type.should.be.eq('dataset') - res.body.result.status.should.be.eq('Draft') - const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA, "transformations_config": [] })) - done(); - }); - }); - - it("Dataset read success: Fetch live dataset when status param is empty", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry") - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.status.should.be.eq('Live') - const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ..._.omit({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "transformations_config": TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA }, ["data_version"]), version: 1 })) - done(); - }); - }); - - it("Dataset read failure: When the dataset of requested dataset_id not found", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=name") - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Dataset with the given dataset_id not found") - res.body.error.code.should.be.eq("DATASET_NOT_FOUND") - done(); - }); - }); - - it("Dataset read failure: When specified field of live dataset cannot be found", (done) => { - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=data") - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - expect(res.body.error.message).to.match(/^The specified field(.+) in the dataset cannot be found.$/) - res.body.error.code.should.be.eq("DATASET_INVALID_FIELDS") - done(); - }); - }); - - it("Dataset read failure: When specified field of draft dataset cannot be found", (done) => { - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=data&status=Draft") - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - expect(res.body.error.message).to.match(/^The specified field(.+) in the dataset cannot be found.$/) - res.body.error.code.should.be.eq("DATASET_INVALID_FIELDS") - done(); - }); - }); - - it("Dataset read failure: Connection to the database failed", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.reject() - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry") - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to read dataset") - res.body.error.code.should.be.eq("DATASET_READ_FAILURE") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts deleted file mode 100644 index 6f20c296..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ /dev/null @@ -1,167 +0,0 @@ -export const TestInputsForDatasetRead = { - DRAFT_SCHEMA: { - "dataset_id": "sb-telemetry", - "id": "sb-telemetry.1", - "name": "sb-telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, - "tags": [ - "tag1", - "tag2" - ], - "status": "Draft", - "version": 1, - "client_state": {}, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "", - "update_date": "", - "published_date": "" - }, - LIVE_SCHEMA: { - - "dataset_id": "sb-telemetry", - "id": "sb-telemetry.1", - "name": "sb-telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, - "tags": [ - "tag1", - "tag2" - ], - "status": "Live", - "data_version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "", - "update_date": "", - "published_date": "" - }, - TRANSFORMATIONS_SCHEMA: [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ] -} \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts deleted file mode 100644 index 7d17e7a7..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ /dev/null @@ -1,213 +0,0 @@ -import app from "../../../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import _ from "lodash"; -import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; -import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("DATASET DENORM UPDATE", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Success: Dataset denorms successfully added", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, denorm_config: { denorm_field: [] } - }) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DENORM_ADD) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: Dataset denorms successfully removed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, denorm_config: { denorm_fields: [{ denorm_out_field: "userdata" }] } - }) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DENORM_REMOVE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: When payload contains same denorms to be removed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", version_key: validVersionKey, status: "Draft", denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_DENORM_REMOVE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - - it("Failure: Dataset contains duplicate denorm field", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ status: "Draft", version_key: validVersionKey }) - }) - chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DUPLICATE_DENORM_KEY) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - expect(res.body.error.message).to.match(/^Dataset contains duplicate denorm out keys(.+)$/) - res.body.error.code.should.be.eq("DATASET_DUPLICATE_DENORM_KEY") - done(); - }); - }); - - it("Failure: When denorm fields provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - }] - } - }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm fields already exist") - res.body.error.code.should.be.eq("DATASET_DENORM_EXISTS") - done(); - }); - }); - - it("Failure: When denorm fields provided to delete does not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "id" - }] - } - }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm fields do not exist to remove") - res.body.error.code.should.be.eq("DATASET_DENORM_DO_NOT_EXIST") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts deleted file mode 100644 index 629eb4ae..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ /dev/null @@ -1,287 +0,0 @@ -import app from "../../../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import _ from "lodash"; -import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("DATASET TRANSFORMATIONS UPDATE", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Success: Dataset transformations successfully added", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_ADD) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: Dataset transformations successfully removed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_REMOVE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: Dataset transformations successfully updated", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: When payload contains same transformation field_key to be added, updated or removed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_TRANSFORMATION_ADD_REMOVE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Failure: When transformation fields provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations already exists") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_EXIST") - done(); - }); - }); - - it("Failure: When transformation fields provided to update do not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key7" }, { field_key: "key2" }]) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations do not exist to update") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_DO_NOT_EXIST") - done(); - }); - }); - - it("Failure: When transformation fields provided to remove do not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key7" }, { field_key: "key3" }]) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations do not exist to remove") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_DO_NOT_EXIST") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/types/ConnectionModels.ts b/api-service/src/v2/types/ConnectionModels.ts deleted file mode 100644 index 02a4d700..00000000 --- a/api-service/src/v2/types/ConnectionModels.ts +++ /dev/null @@ -1,13 +0,0 @@ - -export interface DbConfig { - host: string; - port: string | number; - database: string; - user: string; - password: string; -} - -export interface DbConnectorConfig { - client: string; - connection: DbConfig; -} diff --git a/api-service/src/v2/types/ExhaustModels.ts b/api-service/src/v2/types/ExhaustModels.ts deleted file mode 100644 index 2c2acf4b..00000000 --- a/api-service/src/v2/types/ExhaustModels.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface DateRange { - from: string, - to: string -} diff --git a/api-service/src/v2/types/QueryModels.ts b/api-service/src/v2/types/QueryModels.ts deleted file mode 100644 index 6e5f414b..00000000 --- a/api-service/src/v2/types/QueryModels.ts +++ /dev/null @@ -1,66 +0,0 @@ -export interface IDataSourceRules { - dataset: string; - queryRules: IQueryTypeRules; -} - -export interface ICommonRules { - maxResultThreshold: number; - maxResultRowLimit: number; -} - -export interface IQueryTypeRules { - groupBy: IRules; - scan: IRules; - topN: IRules; - timeseries: IRules; - timeBoundary: IRules; - search: IRules; -} - -export interface IRules { - maxDateRange?: number; -} - -export interface IFilter { - type?: string; - fields?: IFilter[]; - field?: IFilter; - dimension?: string; - dimensions?: string[]; -} - -interface ISqlQueryObject { - query: string; -} - -interface ISqlQuery { - context: object; - querySql: ISqlQueryObject; - query: never; -} - -interface INativeQueryObj { - queryType: string; - dataSource: string; - dimension?: string; - dimensions?: string[]; - filter?: IFilter; - aggregations?: any[]; - postAggregations?: any[]; - granularity: string; - limit?: number; - threshold?: number; - intervals: string[] | string; -} - -interface INativeQuery { - context: object; - query: INativeQueryObj; - querySql: never; -} - -export interface ILimits { - common: ICommonRules; - rules: IDataSourceRules[]; -} -export type IQuery = ISqlQuery | INativeQuery; diff --git a/api-service/src/v2/types/ValidationModels.ts b/api-service/src/v2/types/ValidationModels.ts deleted file mode 100644 index 993e3962..00000000 --- a/api-service/src/v2/types/ValidationModels.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface ValidationStatus { - isValid: boolean, - message?: string, - error?: string, - code?: string -} \ No newline at end of file diff --git a/api-service/src/v2/validators/RequestsValidator.ts b/api-service/src/validators/RequestsValidator.ts similarity index 100% rename from api-service/src/v2/validators/RequestsValidator.ts rename to api-service/src/validators/RequestsValidator.ts diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index ba2ef5d7..8814daac 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: Obsrv API Service + title: Obsrv API V2 description: >- Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as @@ -17,8 +17,11 @@ tags: - name: Data Ingest - name: Data query - name: Data exhaust + - name: Query Templates + - name: Dataset copy + - name: Schema validator paths: - /v1/datasets/create: + /v2/datasets/create: post: tags: - Dataset CRUD APIs @@ -34,12 +37,12 @@ paths: type: object example: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-10T16:10:50+05:30' params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: sb-telemetry + dataset_id: telemetry_record type: dataset name: sb-telemetry validation_config: @@ -58,19 +61,35 @@ paths: $schema: https://json-schema.org/draft/2020-12/schema type: object properties: - eid: + mid: type: string - ver: + arrival_format: text + data_type: string + ets: + type: integer + arrival_format: number + data_type: epoch + eid: type: string - required: - - eid + arrival_format: text + data_type: string additionalProperties: true denorm_config: denorm_fields: - - denorm_key: actor.id + - denorm_key: eid denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid + transformations_config: + - field_key: eid + transformation_function: + type: mask + expr: eid + condition: null + mode: Strict + metadata: + _transformationType: mask + _transformedFieldDataType: string + _transformedFieldSchemaType: string + section: transformation dataset_config: data_key: mid timestamp_key: ets @@ -89,7 +108,7 @@ paths: summary: Dataset Success value: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-16T17:56:06+05:30' params: status: SUCCESS @@ -103,7 +122,7 @@ paths: summary: Master Dataset Success value: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-16T17:56:06+05:30' params: status: SUCCESS @@ -124,31 +143,29 @@ paths: summary: 'Failure: Dataset contains duplicate denorm out field' value: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-16T17:59:06+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 54c5b692-dc37-432e-b556-7f396d7c9e07 responseCode: BAD_REQUEST - result: {} error: - code: DUPLICATE_DENORM_KEY_FOUND + code: DATASET_DUPLICATE_DENORM_KEY message: Duplicate denorm key found trace: '' example-1: summary: 'Failure: Invalid request payload provided' value: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-16T18:00:34+05:30' params: status: FAILED resmsgid: 615c1e4c-8c19-44fd-b29c-c235e7cbb5f0 responseCode: BAD_REQUEST - result: {} error: - code: SCHEMA_VALIDATION_FAILED + code: DATASET_INVALID_INPUT message: >- #additionalProperties should NOT have additional properties @@ -161,14 +178,13 @@ paths: type: object example: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-16T17:56:06+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 76612ad1-241b-4ce3-8af4-88db860697f4 responseCode: CONFLICT - result: {} error: code: DATASET_EXISTS message: Dataset already exists @@ -181,26 +197,26 @@ paths: type: object example: id: api.datasets.create - ver: v1 + ver: v2 ts: '2024-04-16T18:02:44+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: dd1c0e11-fb4c-484c-81fe-82c9e2eee053 responseCode: INTERNAL_SERVER_ERROR - result: {} error: code: DATASET_CREATION_FAILURE message: Failed to create dataset trace: '' - /v1/datasets/update: + /v2/datasets/update: patch: tags: - Dataset CRUD APIs summary: Dataset update description: >- This API allows you to update existing datasets, add or remove denorm - fields used by the analytical data source. User can even add, remove or update transformations + fields used by the analytical data source. User can even add, remove or + update transformations requestBody: required: true content: @@ -209,13 +225,13 @@ paths: type: object example: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-10T16:10:50+05:30' params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: sb-telemetry.1 - version_key: '1713465933736' + dataset_id: telemetry_record + version_key: '1717073955640' name: sb-telemetry validation_config: validate: true @@ -228,14 +244,14 @@ paths: dedup_key: id dedup_config: drop_duplicates: true - dedup_key: mid + dedup_key: eid data_schema: $schema: https://json-schema.org/draft/2020-12/schema type: object properties: eid: type: string - ver: + ets: type: string required: - eid @@ -243,11 +259,11 @@ paths: denorm_config: denorm_fields: - values: - denorm_key: actor.id + denorm_key: eid denorm_out_field: userdata action: remove - values: - denorm_key: actor.id + denorm_key: eid denorm_out_field: edata action: add transformation_config: @@ -265,7 +281,7 @@ paths: section: transformation action: add dataset_config: - data_key: mid + data_key: eid timestamp_key: ets tags: - values: @@ -285,7 +301,7 @@ paths: type: object example: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T00:16:13+05:30' params: status: SUCCESS @@ -307,14 +323,13 @@ paths: summary: 'Failure: Invalid payload provided' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:22:12+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 9408e137-ada8-48b8-99e9-90d5cdd35e35 responseCode: BAD_REQUEST - result: {} error: code: DATASET_UPDATE_INPUT_INVALID message: >- @@ -325,14 +340,13 @@ paths: summary: 'Failure: No field provided along with dataset_id' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:23:06+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 347f465f-f100-4095-9801-38c4943380c0 responseCode: BAD_REQUEST - result: {} error: code: DATASET_UPDATE_NO_FIELDS message: >- @@ -343,14 +357,13 @@ paths: summary: 'Failure: Cannot update as dataset not in draft state' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:25:57+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: a9dfc926-c893-4ec3-82b8-bc8601d928a9 responseCode: BAD_REQUEST - result: {} error: code: DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE message: Dataset cannot be updated as it is not in draft state @@ -359,14 +372,13 @@ paths: summary: 'Failure: Dataset contains duplicate denorm out field' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:29:25+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 865b9bef-9b6d-467c-9065-6689126b0f42 responseCode: BAD_REQUEST - result: {} error: code: DATASET_DUPLICATE_DENORM_KEY message: Dataset contains duplicate denorm out keys:[userdata] @@ -375,14 +387,13 @@ paths: summary: 'Failure: Dataset tags to add already exists' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:32:05+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 9e9ff0bc-d660-4c4e-a2b0-cb11246a6961 responseCode: BAD_REQUEST - result: {} error: code: DATASET_TAGS_EXISTS message: Dataset tags already exist @@ -391,14 +402,13 @@ paths: summary: 'Failure: Dataset transformations to add already exists' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:34:46+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: e0a5cc26-032e-4be8-81ef-c7e76867c9c1 responseCode: BAD_REQUEST - result: {} error: code: DATASET_TRANSFORMATIONS_EXIST message: Dataset transformations already exists @@ -407,14 +417,13 @@ paths: summary: 'Failure: Dataset denorm fields to add already exists' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:38:57+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 0b807580-a2b5-4a44-adc4-80b35b4b6529 responseCode: BAD_REQUEST - result: {} error: code: DATASET_DENORM_EXISTS message: Denorm fields already exist @@ -430,14 +439,13 @@ paths: summary: 'Failure: Dataset does not exists to update' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:20:39+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: ebcf0a62-7c95-40f5-adf2-87ab90a40a80 responseCode: NOT_FOUND - result: {} error: code: DATASET_NOT_EXISTS message: Dataset does not exists to update @@ -446,14 +454,13 @@ paths: summary: 'Failure: Dataset tags to remove do not exist' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:33:33+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: c29e7d0f-3eaf-4972-8804-c172db7b6f13 responseCode: NOT_FOUND - result: {} error: code: DATASET_TAGS_DO_NOT_EXIST message: Dataset tags do not exist to remove @@ -462,14 +469,13 @@ paths: summary: 'Failure: Dataset transformations to update do not exists' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:36:04+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 0b5f515b-1958-4597-b603-b4038d4e5846 responseCode: NOT_FOUND - result: {} error: code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST message: Dataset transformations do not exist to update @@ -478,14 +484,13 @@ paths: summary: 'Failure: Dataset transformations to remove do not exist' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:37:20+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 69d32411-4252-4b7f-8afa-6d436928c3d3 responseCode: NOT_FOUND - result: {} error: code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST message: Dataset transformations do not exist to remove @@ -494,18 +499,38 @@ paths: summary: 'Failure: Dataset denorm to remove does not exist' value: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:40:07+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: ce78e393-2b74-46da-a70b-2abd06ad3270 responseCode: NOT_FOUND - result: {} error: code: DATASET_DENORM_DO_NOT_EXIST message: Denorm fields do not exist to remove trace: '' + '409': + description: Conflict + content: + application/json: + schema: + type: object + example: + id: api.datasets.update + ver: v2 + ts: '2024-04-24T12:30:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: fe4a438d-441c-4497-a3ec-a0cee2058b48 + responseCode: CONFLICT + error: + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates + trace: '' '500': description: Internal Server Error content: @@ -514,33 +539,36 @@ paths: type: object example: id: api.datasets.update - ver: v1 + ver: v2 ts: '2024-04-19T12:41:33+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 451daa85-9cb8-459a-b681-a256c6e912f0 responseCode: INTERNAL_SERVER_ERROR - result: {} error: code: DATASET_UPDATE_FAILURE message: Failed to update dataset trace: '' - /v1/datasets/read/{dataset_id}: + /v2/datasets/read/{dataset_id}: get: tags: - Dataset CRUD APIs summary: Dataset read description: >- - This API allows you to read dataset from the requested dataset_id. - User can request for the specific fields and status of the dataset through the request params. - By default, the API returns the dataset of status "Live". + This API allows you to read dataset from the requested dataset_id. User + can request for the specific fields and status of the dataset through + the request params. By default, the API returns the dataset of status + "Live". + This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, + it creates one using the live dataset and returns the dataset details.

- Valid fields that user can request are "dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date" + Valid fields that user can request are + "dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date" parameters: - name: dataset_id - example: sb_telemetry.1 + example: sb_telemetry in: path required: true schema: @@ -557,6 +585,12 @@ paths: required: false schema: type: string + - name: mode + example: edit + in: query + required: false + schema: + type: string responses: '200': description: OK @@ -569,7 +603,7 @@ paths: summary: Read Draft dataset value: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:43:58+05:30' params: status: SUCCESS @@ -646,7 +680,7 @@ paths: summary: Read Live dataset value: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:44:54+05:30' params: status: SUCCESS @@ -727,7 +761,7 @@ paths: summary: Read specific fields from the dataset value: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:45:44+05:30' params: status: SUCCESS @@ -741,7 +775,7 @@ paths: summary: Read version_key from the dataset value: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:47:48+05:30' params: status: SUCCESS @@ -749,6 +783,333 @@ paths: responseCode: OK result: version_key: '1713510527662' + example-4: + summary: 'Read Dataset: Create a draft dataset on edit live' + value: + id: api.datasets.read + ver: v2 + ts: '2024-06-18T14:44:54+05:30' + params: + status: SUCCESS + resmsgid: 9f8f6a80-f056-424a-97c4-179e2b9199f2 + responseCode: OK + result: + id: master-telemetry + dataset_id: master-telemetry + type: master-dataset + name: master-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: false + extraction_key: '' + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: http://json-schema.org/draft-04/schema# + type: object + properties: + actor: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + eid: + type: string + arrival_format: text + data_type: string + edata: + type: object + properties: + data: + type: object + properties: + oldFirstAccessedOn: + type: string + arrival_format: text + data_type: string + updatedBy: + type: string + arrival_format: text + data_type: string + lastAccessedOn: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + oldLastAccessedOn: + type: string + arrival_format: text + data_type: string + progress: + type: number + arrival_format: number + data_type: number + previousProgress: + type: number + arrival_format: number + data_type: number + contentType: + type: string + arrival_format: text + data_type: string + resourceType: + type: string + arrival_format: text + data_type: string + objectId: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + type: + type: string + arrival_format: text + data_type: string + props: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + ver: + type: string + arrival_format: text + data_type: string + syncts: + type: number + arrival_format: number + data_type: number + '@timestamp': + type: string + arrival_format: text + data_type: string + ets: + type: number + arrival_format: number + data_type: number + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + env: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + flags: + type: object + properties: + ex_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_validation_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_duplicate_skipped: + type: boolean + arrival_format: boolean + data_type: boolean + arrival_format: object + data_type: object + additionalProperties: true + mid: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + additionalProperties: true + dedup_config: + drop_duplicates: false + dedup_key: id + dedup_period: 604800 + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: master-telemetry + dataset_config: + data_key: ver + timestamp_key: obsrv_meta.syncts + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 70 + file_upload_path: [] + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: http://json-schema.org/draft-04/schema# + type: object + properties: + actor: + type: object + properties: + id: + type: string + type: + type: string + required: + - id + - type + eid: + type: string + edata: + type: object + properties: + data: + type: object + properties: + oldFirstAccessedOn: + type: string + updatedBy: + type: string + lastAccessedOn: + type: string + content_id: + type: string + oldLastAccessedOn: + type: string + progress: + type: number + previousProgress: + type: number + contentType: + type: string + resourceType: + type: string + objectId: + type: string + required: + - oldFirstAccessedOn + - updatedBy + - lastAccessedOn + - content_id + - oldLastAccessedOn + - progress + - previousProgress + - contentType + - resourceType + - objectId + type: + type: string + props: + type: string + required: + - data + - type + - props + ver: + type: string + syncts: + type: number + '@timestamp': + type: string + ets: + type: number + context: + type: object + properties: + channel: + type: string + pdata: + type: object + properties: + id: + type: string + required: + - id + env: + type: string + required: + - channel + - pdata + - env + flags: + type: object + properties: + ex_processed: + type: boolean + pp_validation_processed: + type: boolean + pp_duplicate_skipped: + type: boolean + required: + - ex_processed + - pp_validation_processed + - pp_duplicate_skipped + mid: + type: string + type: + type: string + required: + - actor + - eid + - edata + - ver + - syncts + - '@timestamp' + - ets + - context + - flags + - mid + - type + tags: [] + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1718702094143' + version: 1 + status: Draft + api_version: v2 + transformations_config: [] '400': description: Bad Request content: @@ -757,13 +1118,12 @@ paths: type: object example: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:46:03+05:30' params: status: FAILED resmsgid: 302fec39-5070-4464-b02f-56b1d0418147 responseCode: BAD_REQUEST - result: {} error: code: DATASET_INVALID_FIELDS message: >- @@ -778,13 +1138,12 @@ paths: type: object example: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:46:14+05:30' params: status: FAILED resmsgid: cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c responseCode: NOT_FOUND - result: {} error: code: DATASET_NOT_FOUND message: Dataset with the given dataset_id not found @@ -797,18 +1156,17 @@ paths: type: object example: id: api.datasets.read - ver: v1 + ver: v2 ts: '2024-04-19T12:46:34+05:30' params: status: FAILED resmsgid: 29d82020-101d-4506-8fb8-6f3603057064 responseCode: INTERNAL_SERVER_ERROR - result: {} error: code: DATASET_READ_FAILURE message: Failed to read dataset trace: '' - /v1/datasets/list: + /v2/datasets/list: post: tags: - Dataset CRUD APIs @@ -824,7 +1182,7 @@ paths: type: object example: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-10T16:10:50+05:30' params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d @@ -843,7 +1201,7 @@ paths: summary: Dataset list success when no filters provided value: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:54:56+05:30' params: status: SUCCESS @@ -1273,7 +1631,7 @@ paths: summary: Dataset list success when status filter provided as array value: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:55:39+05:30' params: status: SUCCESS @@ -1414,7 +1772,7 @@ paths: summary: Dataset list success when status filter provided as string value: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:56:13+05:30' params: status: SUCCESS @@ -1569,7 +1927,7 @@ paths: summary: Dataset list success when type filter provided value: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:57:28+05:30' params: status: SUCCESS @@ -1653,7 +2011,7 @@ paths: summary: Dataset list success based on sortBy values value: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:58:27+05:30' params: status: SUCCESS @@ -1798,13 +2156,12 @@ paths: type: object example: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:58:47+05:30' params: status: FAILED resmsgid: f9e1fae9-4660-409d-bb4d-e29888983d4b responseCode: BAD_REQUEST - result: {} error: code: DATASET_LIST_INPUT_INVALID message: >- @@ -1819,516 +2176,2376 @@ paths: type: object example: id: api.datasets.list - ver: v1 + ver: v2 ts: '2024-04-19T12:59:15+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 81c3a4a5-e8ac-41de-be00-6d0300ecb106 responseCode: INTERNAL_SERVER_ERROR - result: {} error: code: DATASET_LIST_FAILURE message: Failed to list dataset trace: '' - /444/v1/data/in/sb-telemetry-imported: - post: + /v2/datasets/diff/{dataset_id}: + get: tags: - - Data Ingest - summary: Data ingest - requestBody: - content: - application/json: - schema: - type: object - example: - data: - event: - eid: INTERACT - date: '2022-01-01' - ver: '3.0' - syncts: 1668591949682 - ets: 1668591949682 + - Dataset CRUD APIs + summary: Dataset Diff + description: >- + This API allows you to return the difference between the live and draft + dataset by returning additions, deletions and modifications in the + dataset. parameters: - - name: Content-Type - in: header + - name: dataset_id + example: telemetry + in: path + required: true schema: type: string - example: application/json responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '162' - ETag: - schema: - type: string - example: W/"a2-VX3w1bt3fUXVVCHlq3Stnp7edkc" - Date: - schema: - type: string - example: Mon, 01 Apr 2024 10:11:46 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Data ingest successful(individual) + summary: Data Format Diff value: - id: api.data.in - ver: v1 - ts: 1711966306164 + id: api.datasets.diff + ver: v2 + ts: '2024-05-30T16:55:25+05:30' params: status: SUCCESS - errmsg: '' + resmsgid: c58aa5f7-f182-4dff-9ee9-16f508e53c51 responseCode: OK result: - message: Data ingested successfully + additions: + - type: dataFormat + items: + - name: extraction_key + value: mid + - name: batch_id + value: id + - name: dedup_config.dedup_key + value: id + modifications: + - type: dataFormat + items: + - name: is_batch_event + value: + from: false + to: true + deletions: [] example-1: - summary: Data ingest successfully(batch) + summary: Timestamp and data key diff value: - id: api.data.in - ver: v1 - ts: 1711966352987 + id: api.datasets.diff + ver: v2 + ts: '2024-05-30T17:00:44+05:30' params: status: SUCCESS - errmsg: '' + resmsgid: 8ec5fa57-5105-4793-ae96-3a7e0550dbe9 responseCode: OK result: - message: Data ingested successfully - '400': - description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '195' - ETag: - schema: - type: string - example: W/"c3-s9LhoXtWetfBYQZfs6ljrZ6qzi0" - Date: - schema: - type: string - example: Mon, 01 Apr 2024 10:13:30 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 - content: - application/json: - schema: - type: object - example: - id: api.data.in - ver: v1 - ts: 1711966410766 - params: - status: FAILED - errmsg: >- - #properties/data/oneOf should match exactly one schema in - oneOf - responseCode: BAD_REQUEST - result: {} - '404': - description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '157' - ETag: - schema: - type: string - example: W/"9d-aWMcO0dxAl8AsPQ5hqQD72PP6NU" - Date: - schema: - type: string - example: Mon, 01 Apr 2024 10:14:06 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Dataset not found + additions: [] + modifications: + - type: timestamp + value: + from: syncts + to: ets + - type: dataKey + value: + from: eid + to: mid + deletions: [] + example-2: + summary: Data Schema Diff value: - id: api.data.in - ver: v1 - ts: 1711966446364 + id: api.datasets.diff + ver: v2 + ts: '2024-05-28T15:40:48+05:30' params: - status: FAILED - errmsg: Dataset with id not found - responseCode: BAD_REQUEST - result: {} - example-1: - summary: Entry topic not found + status: SUCCESS + resmsgid: c364416d-b903-4af6-a39b-a4e52c4bf6b5 + responseCode: OK + result: + additions: + - type: dataSchema + items: + - name: eid_1 + value: + arrivalFormat: boolean + dataType: boolean + absolutePath: $.properties.eid_1 + isRequired: false + modifications: + - type: dataSchema + items: + - field: eid + name: type + value: + from: string + to: boolean + - field: eid + name: arrivalFormat + value: + from: text + to: boolean + - field: eid + name: dataType + value: + from: string + to: boolean + deletions: + - type: dataSchema + items: + - name: ver + example-3: + summary: Data Validation Diff value: - id: api.data.in - ver: v1 - ts: 1711975482739 + id: api.datasets.diff + ver: v2 + ts: '2024-05-30T17:05:28+05:30' params: - status: FAILED - errmsg: Entry topic is not defined - responseCode: BAD_REQUEST - result: {} - /444/v1/data/query: + status: SUCCESS + resmsgid: ad6b4977-c341-4193-a42f-9b45993e5f8e + responseCode: OK + result: + additions: [] + modifications: + - type: validation + items: + - field: mode + value: + from: IgnoreNewFields + to: Strict + deletions: [] + example-4: + summary: Dataset Dedup diff + value: + id: api.datasets.diff + ver: v2 + ts: '2024-05-30T17:07:58+05:30' + params: + status: SUCCESS + resmsgid: e59b841b-e35f-462d-adb1-31018d718f6a + responseCode: OK + result: + additions: + - type: dedup + items: + - name: dedup_key + value: mid + modifications: + - type: dedup + items: + - name: drop_duplicates + value: + from: false + to: true + deletions: [] + example-5: + summary: Dataset Denorm Diff + value: + id: api.datasets.diff + ver: v2 + ts: '2024-05-30T17:17:35+05:30' + params: + status: SUCCESS + resmsgid: ee558c17-b3a6-41cc-9b07-64105053394e + responseCode: OK + result: + additions: + - type: denorm + items: + - name: syncts + value: + denorm_key: actor.id + dataset_name: mid + modifications: [] + deletions: + - type: denorm + items: + - name: mid + value: + denorm_key: actor.id + dataset_name: mid + example-6: + summary: Dataset transformation diff + value: + id: api.datasets.diff + ver: v2 + ts: '2024-05-30T17:25:12+05:30' + params: + status: SUCCESS + resmsgid: f4389891-9c09-4ccd-affe-39970590045b + responseCode: OK + result: + additions: + - type: transformations + items: + - name: skill_id + value: + transformation_function: + type: mask + condition: null + expr: skill_id + mode: Strict + metadata: + section: pii + modifications: + - type: transformations + items: + - name: new + field: transformation_function.type + value: + from: jsonata + to: encrypt + metadata: + section: transformation + deletions: + - type: transformations + items: + - name: mid + value: + metadata: + section: transformation + - name: name + value: + metadata: + section: transformation + - name: neew + value: + metadata: + section: additionalFields + /v2/files/generate-url: post: tags: - - Data query - summary: Data query + - Dataset CRUD APIs + summary: Files generate URL + description: >- + This API allows you to generate pre signed urls from cloud providers for + getting and putting files. On access: write, upload urls are generated to put files and on access: read, download urls are generated to get files. requestBody: - content: - application/json: - schema: - type: object - example: - context: - granularity: day - query: >- - SELECT * FROM "rollups-configured" WHERE __time >= TIMESTAMP - '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1 + content: + application/json: + examples: + example1: + summary: Generate upload URL + value: + id: api.files.generate-url + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + example2: + summary: Generate download URL + value: + id: api.files.generate-url + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: read responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '2280' - ETag: - schema: - type: string - example: W/"8e8-TlKxtgzZCmOLRiPGIN2mpTr0eQ4" - Date: - schema: - type: string - example: Mon, 01 Apr 2024 11:46:12 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Success: sql query' + summary: 'Success: Generate Put URL' value: - id: api.data.out - ver: v1 - ts: 1711971972855 + id: api.files.generate-url + ver: v2 + ts: '2024-05-30T12:04:15+05:30' params: status: SUCCESS - errmsg: '' + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: a79f1260-1b34-4b96-9c12-cf0f5600e9f3 responseCode: OK result: - - __time: '2022-01-01T00:00:00.000Z' - eid: '1' - ver: '3.0' - flags_ex_processed: 'true' - flags_pp_validation_processed: 'true' - flags_pp_duplicate_skipped: 'true' - flags_device_denorm: 'true' - flags_user_denorm: 'true' - flags_loc_denorm: 'true' - derivedlocationdata_district: AGRA - derivedlocationdata_from: user-profile - derivedlocationdata_state: Uttar Pradesh - mid: 6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0 - type: events - actor_type: User - actor_id: 311663b2-d7de-4d46-8803-20407eaa3403 - edata_type: session - userdata_subject: null - userdata_district: AGRA - userdata_usersubtype: hm - userdata_grade: null - userdata_usersignintype: Self-Signed-In - userdata_usertype: administrator - userdata_userlogintype: administrator - userdata_state: Uttar Pradesh - devicedata_statecustomcode: '29' - devicedata_country: India - devicedata_iso3166statecode: IN-KA - devicedata_city: Bengaluru - devicedata_countrycode: IN - devicedata_state: Karnataka - devicedata_devicespec_idisk: '106.47' - devicedata_devicespec_webview: 107.0.5304.105 - devicedata_devicespec_os: Android 12 - devicedata_devicespec_scrn: '6.53' - devicedata_devicespec_sims: '-1' - devicedata_devicespec_cpu: "abi: arm64-v8a processor\t: 0 " - devicedata_devicespec_id: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - devicedata_devicespec_camera: '' - devicedata_devicespec_edisk: '106.27' - devicedata_devicespec_make: vivo 1915 - devicedata_statecode: KA - devicedata_districtcustom: BENGALURU URBAN SOUTH - devicedata_statecustomname: Karnataka - devicedata_userdeclared_district: AGRA - devicedata_userdeclared_state: Uttar Pradesh - context_cdata_id: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_cdata_type: UserSession - context_env: sdk - context_channel: '0126796199493140480' - context_pdata_id: preprod.diksha.app - context_pdata_pid: sunbird.app - context_pdata_ver: 4.10.1023preproduction - context_sid: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_did: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - context_rollup_l1: '0126796199493140480' - object_id: '' - object_type: '' - object_version: '' - object_rollup: '{}' - count: 3 - edata_duration_sum: null + - filePath: >- + obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json + fileName: telemetry.json + preSignedUrl: >- + https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=0zWi7R9wqUT6uW0QuN%2F0nyvaloSCqRPTvzHn%2BpkCWdk%3D + - filePath: >- + obsrv-onest/api-service/user_uploads/school_data_14092a.json + fileName: school_data.json + preSignedUrl: >- + https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data_14092a.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=35g%2F46r%2FOI490%2B8pKry9mEbAgXOXznmCyw7L%2FSLITM0%3D example-1: - summary: 'Success: native query' + summary: 'Success: Generate Get URL' value: - id: api.data.out - ver: v1 - ts: 1711973520793 + id: api.files.generate-url + ver: v2 + ts: '2024-05-30T12:05:47+05:30' params: status: SUCCESS - errmsg: '' + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 141717fb-9b94-463d-b0f1-1bab40b187cc responseCode: OK result: - - timestamp: '2022-01-01T00:00:00.000Z' - result: - mid: 1 + - filePath: obsrv-onest/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=UEWKMP9wcETNKA1sXo%2BVI6qZ7Iqk%2BJg6kglaEXz7%2B1Y%3D + - filePath: obsrv-onest/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=jXzootTAoUAWLYKF9AYwKh9Mzl%2FVctlebTBHVz%2BbaKM%3D '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '201' - ETag: - schema: - type: string - example: W/"c9-oV8Psx0gaXpIUOCQT9lFkcsl0SU" - Date: - schema: - type: string - example: Mon, 01 Apr 2024 11:49:53 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: invalid date range' + summary: 'Failure: Invalid access value requested' value: - id: api.data.out - ver: v1 - ts: 1711972193426 + id: api.files.generate-url + ver: v2 + ts: '2024-05-30T12:08:23+05:30' params: status: FAILED - errmsg: >- - Invalid date range! make sure your range cannot be more - than 30 days + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: b1da48a0-5381-49a1-9d5f-ce6c7c65fdcf responseCode: BAD_REQUEST - result: {} + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum should be + equal to one of the allowed values + trace: '' example-1: - summary: 'Failure: Invalid date range (native)' + summary: >- + Failure: Number of file url requested exceeds the limit + configured value: - id: api.data.out - ver: v1 - ts: 1711972501847 + id: api.files.generate-url + ver: v2 + ts: '2024-05-30T12:10:29+05:30' params: status: FAILED - errmsg: >- - Invalid date range! make sure your range cannot be more - than 30 days + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5828abd2-2934-40ef-8fa1-a9bca1e7e24e responseCode: BAD_REQUEST - result: {} - '404': - description: Not Found - headers: - X-Powered-By: + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + /v2/obsrv/data/sql-query: + post: + tags: + - Dataset CRUD APIs + summary: SQL Wrapper + description: This api is a wrapper api, used to query the data. + requestBody: + content: + application/json: + schema: + type: object + example: + query: >- + SELECT "channel","flags", COUNT(*) AS "Count" FROM "wikipedia" + WHERE "page" IS NOT NULL GROUP BY 1, 2 ORDER BY 3 DESC + responses: + '200': + description: OK + content: + application/json: schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '201' - ETag: - schema: - type: string - example: W/"c9-oOPdPjvTI/yRUcA8V6d5riD4+80" - Date: + type: object + example: + - channel: '#en.wikipedia' + flags: '' + Count: 4776 + - channel: '#fr.wikipedia' + flags: '!M' + Count: 31 + - channel: '#fa.wikipedia' + flags: B + Count: 30 + - channel: '#vi.wikipedia' + flags: 'N' + Count: 9 + - channel: '#ar.wikipedia' + flags: '!N' + Count: 8 + - channel: '#da.wikipedia' + flags: '!' + Count: 8 + - channel: '#hy.wikipedia' + flags: '!N' + Count: 8 + - channel: '#id.wikipedia' + flags: '!N' + Count: 8 + - channel: '#eu.wikipedia' + flags: M + Count: 3 + - channel: '#fa.wikipedia' + flags: '!N' + Count: 3 + - channel: '#fi.wikipedia' + flags: 'N' + Count: 3 + - channel: '#he.wikipedia' + flags: NB + Count: 3 + - channel: '#hi.wikipedia' + flags: '' + Count: 1 + - channel: '#hr.wikipedia' + flags: '!' + Count: 1 + - channel: '#hr.wikipedia' + flags: '!N' + Count: 1 + '400': + description: Bad Request + content: + application/json: schema: - type: string - example: Mon, 01 Apr 2024 11:48:05 GMT - Connection: + type: object + example: + id: api.obsrv.data.sql-query + ver: v2 + ts: '2024-05-30T16:31:57+05:30' + params: + status: FAILED + resmsgid: 7709ef5b-d7f3-48a5-bf6c-cf243fb6d2eb + responseCode: BAD_REQUEST + error: + code: SQL_QUERY_EMPTY_REQUEST + message: Failed to query. Invalid request + trace: '' + '500': + description: Internal Server Error + content: + application/json: schema: - type: string - example: keep-alive - Keep-Alive: + type: object + example: + id: api.obsrv.data.sql-query + ver: v2 + ts: '2024-05-29T13:03:44+05:30' + params: + status: FAILED + resmsgid: 0b4b7168-2b70-406b-84ce-c320b65559eb + responseCode: INTERNAL_SERVER_ERROR + error: + code: ERR_BAD_RESPONSE + message: Request failed with status code 500 + trace: '' + /v2/datasets/status-transition: + post: + tags: + - Dataset CRUD APIs + summary: Dataset Status Transition + description: >- + This API allows you to perform status transition between 2 states. + Allowed status transition are: +
+ + Draft to ReadyToPublish when the dataset configuration is complete and configuration is validated, ReadyToPublish to Live to make dataset live and make it actively running within the pipeline, Live to Retired when dataset is needed to be stopped and retired.User can even Delete the dataset of Draft and ReadyToPublish status. + requestBody: + content: + application/json: + examples: + example1: + summary: Transition to ReadyToPublish + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: ReadyToPublish + example2: + summary: Transition to Live + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: Live + example3: + summary: Transition to Retired + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: Retire + example4: + summary: Delete a draft dataset + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: Delete + responses: + '200': + description: OK + content: + application/json: schema: - type: string - example: timeout=5 + type: object + examples: + example-0: + summary: 'Delete success: Deleted dataset successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:18:54+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + responseCode: OK + result: + message: Dataset deleted successfully + dataset_id: master.1 + example-1: + summary: 'Live success: Dataset published successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:21:42+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + responseCode: OK + result: + message: Dataset published successfully + dataset_id: telemetry + example-2: + summary: 'ReadyToPublish success: Dataset is ready to publish' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:30:04+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + responseCode: OK + result: + message: Dataset status transition to ReadyToPublish successful + dataset_id: telemetry-events + example-3: + summary: 'Retire success: Dataset retired successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:22:58+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + responseCode: OK + result: + message: Dataset retired successfully + dataset_id: sb-telemetry + '400': + description: Bad Request content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Datasource not found in druid' + summary: 'Live failure: Dataset in draft state' value: - id: api.data.out - ver: v1 - ts: 1711972085550 + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:37:43+05:30' params: status: FAILED - errmsg: >- - Datasource rollups-configured.1_rollup_week not - available for querying - responseCode: NOT_FOUND - result: {} + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + responseCode: BAD_REQUEST + error: + code: DATASET_NOT_READY_FOR_PUBLISH + message: Failed to publish dataset as it is in draft state + trace: '' example-1: - summary: 'Failure: Datasource not found in live table' + summary: 'ReadyToPublish failure: Incomplete dataset configs' value: - id: api.data.out - ver: v1 - ts: 1711975034302 + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:36:16+05:30' params: status: FAILED - errmsg: Datasource undefined not found - responseCode: NOT_FOUND - result: {} + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + responseCode: BAD_REQUEST + error: + code: DATASET_CONFIGS_INVALID + message: >- + #properties/denorm_config/properties/denorm_fields/items/required + must have required property 'dataset_name' + trace: '' example-2: - summary: 'Failure: Datasource not found in live table (native)' + summary: 'ReadyToPublish failure: Dataset not in draft state' value: - id: api.data.out - ver: v1 - ts: 1711974978635 + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:38:14+05:30' params: status: FAILED - errmsg: Datasource telemetry-eventssss not found - responseCode: NOT_FOUND - result: {} + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + responseCode: BAD_REQUEST + error: + code: DATASET_READYTOPUBLISH_FAILURE + message: >- + Failed to mark dataset Ready to publish as it not in + draft state + trace: '' example-3: - summary: 'Failure: Datasource not found in druid(native)' + summary: 'Retire Failure: Dataset is already retired' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:42:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 + responseCode: BAD_REQUEST + error: + code: DATASET_ALREADY_RETIRED + message: Dataset is already retired + trace: '' + example-4: + summary: >- + Retire failure: Cannot retire master dataset as it is used + by other datasets + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:01:41+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 + responseCode: BAD_REQUEST + error: + code: DATASET_IN_USE + message: Failed to retire dataset as it is used by other datasets + trace: '' + example-5: + summary: 'Failure: Invalid request payload provided' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:03:56+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 + responseCode: BAD_REQUEST + error: + code: DATASET_STATUS_INVALID_INPUT + message: >- + #properties/request/properties/status/enum should be + equal to one of the allowed values + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete failure: Dataset not found to delete' value: - id: api.data.out - ver: v1 - ts: 1711972569115 + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:25:36+05:30' params: status: FAILED - errmsg: >- - Datasource telemetry-events.1_rollup_day not available - for querying + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b responseCode: NOT_FOUND - result: {} - /444/v1/data/exhaust/beckn-test-data: - get: - tags: - - Data exhaust - summary: Data exhaust - parameters: - - name: from - in: query - schema: - type: string - example: '2023-10-01' - - name: to - in: query + error: + code: DATASET_NOT_FOUND + message: Dataset not found to delete + trace: '' + example-1: + summary: 'Live failure: Dataset not found to publish' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:35:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: Dataset not found to publish + trace: '' + example-2: + summary: 'Retire Failure: Dataset not found to retire' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:40:31+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: Dataset not found to retire + trace: '' + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + example: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:24:12+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 92928434-719f-47d4-9946-1e40ecd53253 + responseCode: INTERNAL_SERVER_ERROR + error: + code: DATASET_STATUS_FAILURE + message: Failed to perform status transition on datasets + trace: '' + /v2/data/in/{datasetId}: + post: + tags: + - Data Ingest + parameters: + - required: true schema: + title: datasetId type: string - example: '2024-03-10' - - name: type - in: query + name: datasetId + in: path + summary: Data ingest + description: >- + This API allows you to ingest events to a data streaming platform in individual or batch format. + requestBody: + content: + application/json: + examples: + example1: + summary: Ingest individual event + value: + id: api.data.in + ver: v2 + ts: "1711966306164" + params: + msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + data: + eid: INTERACT + date: '2022-01-01' + ver: '3.0' + syncts: 1668591949682 + ets: 1668591949682 + example2: + summary: Ingest batch event + value: + id: api.data.in + ver: v2 + ts: "1711966306164" + params: + msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + data: [{ "eid": "INTERACT","date": "2022-01-01","ver": "3.0","syncts": 1668591949682}] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: Data ingest successful(individual) + value: + id: api.data.in + ver: v2 + ts: 1711966306164 + params: + status: "SUCCESS" + msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" + responseCode: OK + result: + message: Data ingested successfully + example-1: + summary: Data ingest successfully(batch) + value: + id: api.data.in + ver: v2 + ts: 1711966352987 + params: + status: "SUCCESS" + msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" + responseCode: OK + result: + message: Data ingested successfully + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: 1711966410766 + params: + status: FAILED + resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 + responseCode: BAD_REQUEST + error: + code: DATA_INGESTION_INVALID_INPUT + message: "#required should have required property 'id'" + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: Dataset not found + value: + id: api.data.in + ver: v2 + ts: 1711966446364 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 + responseCode: BAD_REQUEST + error: + code: DATASET_NOT_FOUND + message: Dataset with id not found + trace: '' + example-1: + summary: Entry topic not found + value: + id: api.data.in + ver: v2 + ts: 1711975482739 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 + responseCode: BAD_REQUEST + error: + code: ENTRY_TOPIC_NOT_FOUND + message: Entry topic is not defined + trace: '' + /v2/data/query/{datasetId}: + post: + tags: + - Data query + summary: Data query + parameters: + - required: true schema: + title: datasetId type: string - example: raw + name: datasetId + in: path + description: >- + This API allows you to query your datasource. + requestBody: + content: + application/json: + examples: + example1: + summary: SQL query + value: + id: api.data.out + ver: v2 + ts: "1711966306164" + params: + msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + context: + datasetId: rollups-configured + table: day + query: >- + SELECT * FROM "rollups-configured" WHERE __time >= TIMESTAMP + '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1 + example2: + summary: Native query + value: + id: api.data.out + ver: v2 + ts: "1711966306164" + params: + mid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + context: + datasetId: rollups-configured + table: day + query: + quertType: scan + dataSource: rollups-configured + intervals: + type: intervals + intervals: ["2022-01-01/2022-02-01"] + granularity: day + columns: ["eid","__time"] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: sql query' + value: + id: api.data.out + ver: v2 + ts: 1711971972855 + params: + status: "SUCCESS" + msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" + resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" + responseCode: OK + result: + - __time: '2022-01-01T00:00:00.000Z' + eid: '1' + ver: '3.0' + flags_ex_processed: 'true' + flags_pp_validation_processed: 'true' + flags_pp_duplicate_skipped: 'true' + flags_device_denorm: 'true' + flags_user_denorm: 'true' + flags_loc_denorm: 'true' + derivedlocationdata_district: AGRA + derivedlocationdata_from: user-profile + derivedlocationdata_state: Uttar Pradesh + mid: 6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0 + type: events + actor_type: User + actor_id: 311663b2-d7de-4d46-8803-20407eaa3403 + edata_type: session + userdata_subject: null + userdata_district: AGRA + userdata_usersubtype: hm + userdata_grade: null + userdata_usersignintype: Self-Signed-In + userdata_usertype: administrator + userdata_userlogintype: administrator + userdata_state: Uttar Pradesh + devicedata_statecustomcode: '29' + devicedata_country: India + devicedata_iso3166statecode: IN-KA + devicedata_city: Bengaluru + devicedata_countrycode: IN + devicedata_state: Karnataka + devicedata_devicespec_idisk: '106.47' + devicedata_devicespec_webview: 107.0.5304.105 + devicedata_devicespec_os: Android 12 + devicedata_devicespec_scrn: '6.53' + devicedata_devicespec_sims: '-1' + devicedata_devicespec_cpu: "abi: arm64-v8a processor\t: 0 " + devicedata_devicespec_id: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 + devicedata_devicespec_camera: '' + devicedata_devicespec_edisk: '106.27' + devicedata_devicespec_make: vivo 1915 + devicedata_statecode: KA + devicedata_districtcustom: BENGALURU URBAN SOUTH + devicedata_statecustomname: Karnataka + devicedata_userdeclared_district: AGRA + devicedata_userdeclared_state: Uttar Pradesh + context_cdata_id: a3c784f0-61d8-43e4-a92a-373fd4338c1d + context_cdata_type: UserSession + context_env: sdk + context_channel: '0126796199493140480' + context_pdata_id: preprod.diksha.app + context_pdata_pid: sunbird.app + context_pdata_ver: 4.10.1023preproduction + context_sid: a3c784f0-61d8-43e4-a92a-373fd4338c1d + context_did: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 + context_rollup_l1: '0126796199493140480' + object_id: '' + object_type: '' + object_version: '' + object_rollup: '{}' + count: 3 + edata_duration_sum: null + example-1: + summary: 'Success: native query' + value: + id: api.data.out + ver: v2 + ts: 1711973520793 + params: + status: SUCCESS + errmsg: '' + responseCode: OK + result: + - timestamp: '2022-01-01T00:00:00.000Z' + result: + mid: 1 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: invalid date range (sql)' + value: + id: api.data.out + ver: v2 + ts: 1711972193426 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 + responseCode: BAD_REQUEST + error: + code: INVALID_DATE_RANGE + message: Invalid date range! make sure your range cannot be more than 30 days + trace: "" + example-1: + summary: 'Failure: Invalid date range (native)' + value: + id: api.data.out + ver: v2 + ts: 1711972501847 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 + responseCode: BAD_REQUEST + error: + code: INVALID_DATE_RANGE + message: Invalid date range! make sure your range cannot be more than 30 days + trace: "" + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Datasource not found in druid' + value: + id: api.data.out + ver: v2 + ts: 1711972085550 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 + responseCode: BAD_REQUEST + error: + code: DATASOURCE_NOT_FOUND + message: Datasource taxt_trip not found + trace: "" + example-1: + summary: 'Failure: Datasource not found in live table' + value: + id: api.data.out + ver: v2 + ts: 1711975034302 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 + responseCode: BAD_REQUEST + error: + code: DATASOURCE_NOT_FOUND + message: Datasource taxt_trip not found + trace: "" + example-2: + summary: 'Failure: Datasource not found in live table (native)' + value: + id: api.data.out + ver: v2 + ts: 1711974978635 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 + responseCode: BAD_REQUEST + error: + code: DATASOURCE_NOT_FOUND + message: Datasource taxt_trip.1_day not available for querying + trace: "" + example-3: + summary: 'Failure: Datasource not found in druid(native)' + value: + id: api.data.out + ver: v2 + ts: 1711972569115 + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 + responseCode: BAD_REQUEST + error: + code: DATASOURCE_NOT_FOUND + message: Datasource taxt_trip.1_day not available for querying + trace: "" + /v2/template/read/{template_id}: + get: + tags: + - Query Templates + summary: Read template + parameters: + - required: true + schema: + title: template_id + type: string + name: template_id + in: path + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: JSON template' + value: + id: api.query.template.read + ver: v2 + ts: '2024-04-30T12:09:55+05:30' + params: + status: SUCCESS + resmsgid: 8ef3a9d0-8658-47dd-b6cb-a4b1777a1ea4 + responseCode: OK + result: + template_id: jsontemplate111 + template_name: jsontemplate111 + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:16:57.023Z' + updated_date: '2024-04-29T01:16:57.023Z' + example-1: + summary: 'Success: SQL template' + value: + id: api.query.template.read + ver: v2 + ts: '2024-04-30T12:10:54+05:30' + params: + status: SUCCESS + resmsgid: 656185ef-527d-4b60-ac19-263da85142b8 + responseCode: OK + result: + template_id: sql1 + template_name: sql1 + query: >- + "SELECT * FROM {{DATASET}} WHERE __time BETWEEN + TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T11:29:58.759Z' + updated_date: '2024-04-29T11:29:58.759Z' + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.query.template.read + ver: v2 + ts: '2024-04-30T12:11:45+05:30' + params: + status: FAILED + resmsgid: bc6a4621-4ac5-4959-b467-1134ea874960 + responseCode: NOT_FOUND + error: + code: QUERY_TEMPLATE_NOT_EXISTS + message: Template sql100 does not exists + trace: '' + /v2/template/delete/{template_id}: + delete: + tags: + - Query Templates + summary: Delete template + parameters: + - required: true + schema: + title: template_id + type: string + name: template_id + in: path + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: api.query.template.delete + ver: v2 + ts: '2024-04-30T11:58:27+05:30' + params: + status: SUCCESS + resmsgid: 3e2859df-f494-4c47-ae64-e2c34f4ef1cb + responseCode: OK + result: + message: Template yashash-k deleted successfully + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.query.template.delete + ver: v2 + ts: '2024-04-29T18:29:09+05:30' + params: + status: FAILED + resmsgid: 6c62ff4d-0a63-4384-a365-7bbef951f288 + responseCode: NOT_FOUND + error: + code: QUERY_TEMPLATE_NOT_EXISTS + message: Template json_template does not exists + trace: '' + /v2/template/create/{template_id}: + post: + tags: + - Query Templates + summary: Create query template + parameters: + - required: true + schema: + title: template_id + type: string + name: template_id + in: path + requestBody: + content: + application/json: + schema: + type: object + example: + id: query.template.create + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + query_type: sql + query: + queryType: select + datasetId: '{{DATASET}}' + intervals: '{{STARTDATE}}/{{ENDDATE}}' + limit: '{{LIMITS}}' responses: '200': - description: Successful response + description: OK content: - application/json: {} + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success : Create query template success (json)' + value: + id: api.query.template.create + ver: v2 + ts: '2024-04-29T22:29:01+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 71372ce0-16b9-4594-8db8-f12eff7e6a42 + responseCode: OK + result: + template_id: json11template + template_name: json11template + message: The query template has been saved successfully + example-1: + summary: 'Success: Create query template success (SQL)' + value: + id: api.query.template.create + ver: v2 + ts: '2024-04-29T22:29:58+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388 + responseCode: OK + result: + template_id: sql1 + template_name: sql1 + message: The query template has been saved successfully + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Missing required variables' + value: + id: api.query.template.create + ver: v2 + ts: '2024-04-29T22:35:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 62e18342-7e25-4122-8fca-6fb12fac3ff0 + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID + message: >- + Invalid template provided, A template should consist of + variables undefined and type of json,sql + trace: '' + example-1: + summary: 'Failure: Schema validation failure' + value: + id: api.query.template.create + ver: v2 + ts: '2024-04-29T22:43:57+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2 + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/oneOf/0/properties/query_type/enum + should be equal to one of the allowed values + trace: '' + '409': + description: Conflict + content: + application/json: + schema: + type: object + example: + id: api.query.template.create + ver: v2 + ts: '2024-04-29T22:33:28+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 18b6b123-4df5-4124-b6ec-73b667250e1c + responseCode: CONFLICT + error: + code: QUERY_TEMPLATE_ALREADY_EXISTS + message: Template josnaks-aaa already exists + trace: '' + /v2/template/list: + post: + tags: + - Query Templates + summary: List templates + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + query_type: sql + sortBy: + - field: created_date + order: desc + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: empty request body + value: + id: api.query.template.list + ver: v2 + ts: '2024-04-29T19:16:03+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 5d41ee6e-2fc6-4353-b6c4-49e068f39b2f + responseCode: OK + result: + - template_id: josnaksaaa + template_name: JOSnaks--aaa + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:25:04.706Z' + updated_date: '2024-04-25T06:25:04.706Z' + - template_id: josnaks-aaa + template_name: JOSnaks--aaa + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:26:35.749Z' + updated_date: '2024-04-25T06:26:35.749Z' + - template_id: a + template_name: ' a' + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:30:50.179Z' + updated_date: '2024-04-25T06:30:50.179Z' + - template_id: yash-k + template_name: yash k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:08.821Z' + updated_date: '2024-04-25T06:31:08.821Z' + - template_id: yashas-k + template_name: yashas k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:20.819Z' + updated_date: '2024-04-25T06:31:20.819Z' + - template_id: yashash-k + template_name: YASHASH k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:39.490Z' + updated_date: '2024-04-25T06:31:39.490Z' + - template_id: yashash-ak + template_name: YASHASH ak + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:33:56.501Z' + updated_date: '2024-04-25T06:33:56.501Z' + - template_id: test_template + template_name: test template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-26T00:24:34.435Z' + updated_date: '2024-04-26T00:24:34.435Z' + - template_id: jsontemplate + template_name: jsontemplate + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-28T23:27:32.113Z' + updated_date: '2024-04-28T23:27:32.113Z' + - template_id: jsontemplate1 + template_name: jsontemplate1 + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-28T23:28:35.868Z' + updated_date: '2024-04-28T23:28:35.868Z' + - template_id: jsontemplate111 + template_name: jsontemplate111 + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:16:57.023Z' + updated_date: '2024-04-29T01:16:57.023Z' + - template_id: jso_template + template_name: jso template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:19:42.458Z' + updated_date: '2024-04-29T01:19:42.458Z' + - template_id: json_1template + template_name: json 1template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:22:19.689Z' + updated_date: '2024-04-29T01:22:19.689Z' + - template_id: sql_template + template_name: sql template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:24:03.511Z' + updated_date: '2024-04-29T01:24:03.511Z' + - template_id: sql1template + template_name: sql1template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:26:28.381Z' + updated_date: '2024-04-29T01:26:28.381Z' + - template_id: sql11template + template_name: sql11template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:30:59.381Z' + updated_date: '2024-04-29T01:30:59.381Z' + - template_id: sql11template1 + template_name: sql11template1 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:31:53.976Z' + updated_date: '2024-04-29T01:31:53.976Z' + - template_id: sql_template_1 + template_name: sql template 1 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:36:01.592Z' + updated_date: '2024-04-29T01:36:01.592Z' + - template_id: sql_template_11 + template_name: sql template 11 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:39:29.968Z' + updated_date: '2024-04-29T01:39:29.968Z' + example-1: + summary: 'Success : order by' + value: + id: api.query.template.list + ver: v2 + ts: '2024-04-30T11:36:09+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 4393ac57-d441-4be8-b22b-9e4328cab887 + responseCode: OK + result: + - template_id: sql11template + template_name: sql11template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-21T01:30:59.381Z' + updated_date: '2024-04-21T01:30:59.381Z' + - template_id: sql11template1 + template_name: sql11template1 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-22T01:31:53.976Z' + updated_date: '2024-04-22T01:31:53.976Z' + - template_id: josnaksaaa + template_name: JOSnaks--aaa + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:25:04.706Z' + updated_date: '2024-04-25T06:25:04.706Z' + - template_id: josnaks-aaa + template_name: JOSnaks--aaa + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:26:35.749Z' + updated_date: '2024-04-25T06:26:35.749Z' + - template_id: a + template_name: ' a' + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:30:50.179Z' + updated_date: '2024-04-25T06:30:50.179Z' + - template_id: yash-k + template_name: yash k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:08.821Z' + updated_date: '2024-04-25T06:31:08.821Z' + - template_id: yashas-k + template_name: yashas k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:20.819Z' + updated_date: '2024-04-25T06:31:20.819Z' + - template_id: yashash-k + template_name: YASHASH k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:39.490Z' + updated_date: '2024-04-25T06:31:39.490Z' + - template_id: yashash-ak + template_name: YASHASH ak + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:33:56.501Z' + updated_date: '2024-04-25T06:33:56.501Z' + - template_id: test_template + template_name: test template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-26T00:24:34.435Z' + updated_date: '2024-04-26T00:24:34.435Z' + - template_id: jsontemplate + template_name: jsontemplate + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-28T23:27:32.113Z' + updated_date: '2024-04-28T23:27:32.113Z' + - template_id: jsontemplate1 + template_name: jsontemplate1 + query: >- + {"queryType":"timeseries","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-28T23:28:35.868Z' + updated_date: '2024-04-28T23:28:35.868Z' + - template_id: jsontemplate111 + template_name: jsontemplate111 + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:16:57.023Z' + updated_date: '2024-04-29T01:16:57.023Z' + - template_id: jso_template + template_name: jso template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:19:42.458Z' + updated_date: '2024-04-29T01:19:42.458Z' + - template_id: json_1template + template_name: json 1template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:22:19.689Z' + updated_date: '2024-04-29T01:22:19.689Z' + - template_id: sql_template + template_name: sql template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:24:03.511Z' + updated_date: '2024-04-29T01:24:03.511Z' + - template_id: sql1template + template_name: sql1template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:26:28.381Z' + updated_date: '2024-04-29T01:26:28.381Z' + - template_id: sql_template_1 + template_name: sql template 1 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:36:01.592Z' + updated_date: '2024-04-29T01:36:01.592Z' + - template_id: sql_template_11 + template_name: sql template 11 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:39:29.968Z' + updated_date: '2024-04-29T01:39:29.968Z' + - template_id: json11template + template_name: json11template + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T11:29:01.096Z' + updated_date: '2024-04-29T11:29:01.096Z' + - template_id: sql1 + template_name: sql1 + query: >- + "SELECT * FROM {{DATASET}} WHERE __time BETWEEN + TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T11:29:58.759Z' + updated_date: '2024-04-29T11:29:58.759Z' + example-2: + summary: 'Success : Filters' + value: + id: api.query.template.list + ver: v2 + ts: '2024-04-30T11:38:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 69da1ef2-c2c5-4f22-bb68-abdf823f0744 + responseCode: OK + result: + - template_id: sql_template + template_name: sql template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:24:03.511Z' + updated_date: '2024-04-29T01:24:03.511Z' + - template_id: sql1template + template_name: sql1template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:26:28.381Z' + updated_date: '2024-04-29T01:26:28.381Z' + - template_id: sql_template_1 + template_name: sql template 1 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:36:01.592Z' + updated_date: '2024-04-29T01:36:01.592Z' + - template_id: sql_template_11 + template_name: sql template 11 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T01:39:29.968Z' + updated_date: '2024-04-29T01:39:29.968Z' + - template_id: sql11template + template_name: sql11template + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-21T01:30:59.381Z' + updated_date: '2024-04-21T01:30:59.381Z' + - template_id: sql11template1 + template_name: sql11template1 + query: >- + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP \"{{STARTDATE}}\" AND TIMESTAMP + \"{{ENDDATE}}\"" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-22T01:31:53.976Z' + updated_date: '2024-04-22T01:31:53.976Z' + - template_id: sql1 + template_name: sql1 + query: >- + "SELECT * FROM {{DATASET}} WHERE __time BETWEEN + TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + query_type: sql + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-29T11:29:58.759Z' + updated_date: '2024-04-29T11:29:58.759Z' + example-3: + summary: 'Success : limit and offset' + value: + id: api.query.template.list + ver: v2 + ts: '2024-04-30T11:41:36+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: d1aa35c3-e817-4e2e-85f5-dfd346122192 + responseCode: OK + result: + - template_id: josnaksaaa + template_name: JOSnaks--aaa + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:25:04.706Z' + updated_date: '2024-04-25T06:25:04.706Z' + - template_id: josnaks-aaa + template_name: JOSnaks--aaa + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:26:35.749Z' + updated_date: '2024-04-25T06:26:35.749Z' + - template_id: a + template_name: ' a' + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:30:50.179Z' + updated_date: '2024-04-25T06:30:50.179Z' + - template_id: yash-k + template_name: yash k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:08.821Z' + updated_date: '2024-04-25T06:31:08.821Z' + - template_id: yashas-k + template_name: yashas k + query: >- + {"queryType":"select","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMITS}}"} + query_type: json + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-04-25T06:31:20.819Z' + updated_date: '2024-04-25T06:31:20.819Z' + /v2/template/update/{template_id}: + patch: + tags: + - Query Templates + summary: update template + parameters: + - required: true + schema: + title: template_id + type: string + name: template_id + in: path + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.update + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + template_name: sql_update_test_template + query_type: sql + query: >- + SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP + {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}} + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: api.query.template.update + ver: v2 + ts: '2024-05-02T13:16:31+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: ab869754-d2f0-4b9a-b38b-04d9e5901b5d + responseCode: OK + result: + message: Query template updated successfully + templateId: sql11template1 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: query should present when query_type provided' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-02T11:30:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f11f5287-77cd-43f0-b7f6-6d6b94b9cea2 + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query when property query_type is present + trace: '' + example-1: + summary: 'Failure: query_type should present when query is given' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-02T11:33:23+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 937cf790-867b-450b-afc9-86a96d02f636 + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query_type when property query is present + trace: '' + example-2: + summary: 'Failure: template name validation failure' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-02T11:34:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9f3ae5aa-2e98-4204-8f87-b11e5bb3865a + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Template name should contain alphanumeric characters and + single space between characters + trace: '' + example-3: + summary: 'Failure: required template_id' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-02T11:38:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 94b71f80-bd1e-4283-8f3d-83833b60d08c + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/required should have required + property 'template_id' + trace: '' + example-4: + summary: 'Failure: required variables not exists' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-02T13:13:22+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: dff3c22a-c198-4d1c-b87a-07d725f868f7 + responseCode: BAD_REQUEST + error: + code: QUERY_TEMPLATE_INVALID + message: >- + Invalid template provided, A template should consist of + variables DATASET,STARTDATE,ENDDATE and type of json,sql + trace: '' + /v2/template/query/{template_id}: + post: + tags: + - Query Templates + summary: query template + parameters: + - required: true + schema: + title: template_id + type: string + name: template_id + in: path + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.query + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + startdate: '2020-12-31' + enddate: '2024-12-31' + aggregationLevel: month + dataset: test + limit: 5 + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: JSON template with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:52:24+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + responseCode: OK + result: + - timestamp: '2023-09-01T00:00:00.000Z' + result: + school_id: 0 + example-1: + summary: 'Success: SQL template query with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:33:06+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: invalid date range (native template)' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-02T12:59:14+05:30' + params: + status: FAILED + resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 + responseCode: BAD_REQUEST + error: + code: DATA_OUT_INVALID_DATE_RANGE + message: >- + Invalid date range! make sure your range cannot be more + than 30 days + trace: '' + example-1: + summary: 'Failure: invalid date range' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-13T13:28:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 + responseCode: BAD_REQUEST + error: + code: DATA_OUT_INVALID_DATE_RANGE + message: >- + Invalid date range! make sure your range cannot be more + than 30 days + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Datasource not found in druid' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-13T13:21:46+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: b35a7050-b94c-4944-9630-233c9542272e + responseCode: NOT_FOUND + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table hour is not available for + querying + trace: '' + example-1: + summary: 'Failure: Datasource not found in live table' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-13T13:23:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 + responseCode: NOT_FOUND + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource test11 not available for querying + trace: '' + /v2/dataset/copy: + post: + tags: + - Dataset copy + summary: Dataset copy + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: '1.0' + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + source: + datasetId: test_dataset + isLive: true + destination: + datasetId: test-dataset-copy-1 + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: api.dataset.copy + ver: v2 + ts: '2024-05-31T10:57:11+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 77080c18-67b3-489c-af7b-9739709e3c4b + responseCode: OK + result: + dataset_id: sample1_copy.1 + message: Dataset clone successful + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.dataset.copy + ver: v2 + ts: '2024-05-31T10:56:26+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: '' + responseCode: BAD_REQUEST + error: + code: DATASET_ALREADY_EXISTS + message: Dataset with id sample1 already exists + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.dataset.copy + ver: v2 + ts: '2024-05-31T10:59:59+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: '' + responseCode: NOT_FOUND + error: + code: DATASET_NOT_EXISTS + message: Dataset sample does not exists + trace: '' diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml new file mode 100644 index 00000000..c5f234a1 --- /dev/null +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -0,0 +1,4893 @@ +openapi: 3.0.0 +info: + title: Obsrv api documentation + version: 1.0.0 +servers: + - url: http://localhost:3000 +tags: + - name: Dataset api's +paths: + /v2/datasets/create: + post: + tags: + - Dataset api's + summary: Dataset create + requestBody: + content: + application/json: + examples: + example1: + summary: Request Body + value: + id: api.datasets.create + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + type: event + name: sb-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_config: + drop_duplicates: true + dedup_key: mid + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + mid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + arrival_format: number + data_type: epoch + eid: + type: string + arrival_format: text + data_type: string + additionalProperties: true + denorm_config: + denorm_fields: + - denorm_key: eid + denorm_out_field: userdata + dataset_id: master-telemetry + transformations_config: + - field_key: email + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + tags: + - tag1 + example2: + summary: Minimal Request Body + value: + id: api.datasets.create + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_events, + type: event + name: sb-telemetry + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Dataset created successfullly' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-15T18:44:08+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 276c042c-0f23-4b26-9b10-6fe48bbc2d3a + responseCode: OK + result: + id: telemetry_record-t4 + version_key: '1721049248930' + example-1: + summary: 'Success: Master dataset created successfully' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:36:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 845076be-d9e7-4246-bb8e-07ae0ce59d1e + responseCode: OK + result: + id: telemetry_record-master + version_key: '1721099200603' + example-2: + summary: 'Success: Minimal request body' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T18:14:59+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9e207f4f-2be6-4a45-ab78-213bea272ae0 + responseCode: OK + result: + id: telemetry_events + version_key: '1721133899306' + example-3: + summary: 'Success: Dataset created successfully with all fields' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-17T18:19:53+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 505fb3bc-ae32-4f5b-a931-adec4d1d84ba + responseCode: OK + result: + id: telemetry_record-t41 + version_key: '1721220593027' + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:39:00+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a07de860-dcbc-4ff6-822e-34b47635c8a3 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + '409': + description: Conflict + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Master dataset already exists' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:37:28+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 138b796b-1b68-481a-a59d-1cb695c1adc9 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-master + example-1: + summary: 'Failure: Dataset already exists' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:38:05+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf62693c-3aa4-42ce-a5ea-4bde340740f5 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-t4 + /v2/datasets/update: + patch: + tags: + - Dataset api's + summary: Dataset update + requestBody: + content: + application/json: + examples: + example1: + summary: Request Body + value: + id: api.datasets.update + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_record-t4 + version_key: '1721135455988' + name: sb-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: ipid + dedup_config: + drop_duplicates: true + dedup_key: mid + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + midpid: + type: string + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + additionalProperties: true + denorm_config: + denorm_fields: + - value: + denorm_key: eid + denorm_out_field: userdata + action: remove + - value: + denorm_key: eid + denorm_out_field: edata + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert + - value: + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + action: remove + tags: [] + connectors_config: + - value: + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice + action: upsert + example2: + summary: Minimal request body + value: + id: api.datasets.update + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_record-t4 + version_key: '1721135455988' + name: sb-telemetry + validation_config: + validate: true + mode: Strict + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Minimal dataset update' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:30:45+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 354f1fec-0c39-42ee-a52a-49552f847c11 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134845559' + example-1: + summary: 'Success: Updated successfully' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:27:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 6d835f07-aed5-4e8b-81c2-2142cfb55c52 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134675878' + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:29:21+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 7d31672b-e5c3-4a6d-afac-d9d78011bcde + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_UPDATE_INPUT_INVALID + message: >- + #properties/request/required must have required property + 'dataset_id' + example-1: + summary: 'Failure: No fields are provided to update' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:32:44+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf99b1e1-7694-4be0-ba5d-e347764736de + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_UPDATE_NO_FIELDS + message: >- + Provide atleast one field in addition to the dataset_id + to update the dataset + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:28:30+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf64703c-bb6b-41bf-bc1a-c85373efd925 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_EXISTS + message: Dataset does not exists with id:telemetry_record-t41 + '409': + description: Conflict + content: + application/json: + schema: + type: object + example: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:23:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fe03f6-c4c4-48f6-9d84-a32cd52f4c13 + responseCode: CONFLICT + result: {} + error: + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates + /v2/datasets/read/{dataset_id}: + get: + tags: + - Dataset api's + summary: Read dataset + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + example: beckn-test-data + - name: mode + in: query + schema: + type: string + example: edit + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Read live dataset' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:38:55+05:30' + params: + status: SUCCESS + resmsgid: 8c8a2852-54bc-43fb-b063-7f359d11930a + responseCode: OK + result: + dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + example-1: + summary: 'Success: Read draft dataset' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:41:00+05:30' + params: + status: SUCCESS + resmsgid: 96fd4f42-fa84-4730-bc79-d241a4e335a1 + responseCode: OK + result: + dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + example-2: + summary: 'Success: Read specific column' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:42:16+05:30' + params: + status: SUCCESS + resmsgid: 02a6b03a-8bf3-4e37-8dcd-859d3e8f904e + responseCode: OK + result: + name: master-test + type: master + id: master-test + example-3: + summary: 'Success: Read version_key' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:45:37+05:30' + params: + status: SUCCESS + resmsgid: 805e848a-d260-47c3-b55c-fc9b8323719e + responseCode: OK + result: + version_key: '1718791650227' + example-4: + summary: >- + Success: Read from draft, if not present cerate draft from + live dataset and then read + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:49:28+05:30' + params: + status: SUCCESS + resmsgid: da70e25b-6ad0-49a7-a39d-340d1d0c46a7 + responseCode: OK + result: + dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 2 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + data_key: '' + timestamp_key: time + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:50:17+05:30' + params: + status: FAILED + resmsgid: bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_FIELDS + message: >- + The specified fields [newname] in the dataset cannot be + found. + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:51:12+05:30' + params: + status: FAILED + resmsgid: 3aad3842-a76e-4fe8-b635-c7fef5f318f9 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_FOUND + message: >- + Dataset with the given dataset_id:new_telemetry_record.1 not + found + /v2/datasets/list: + post: + tags: + - Dataset api's + summary: Dataset list + requestBody: + content: + application/json: + examples: + example1: + summary: Filter based on status of type array + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Live + example2: + summary: No filters provided + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: {} + example3: + summary: Filter based on status of type string + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: ReadyToPublish + example4: + summary: Filter based on dataset type + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: Live + type: master + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Lists all when no filters provided' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T17:55:36+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 97efe04d-e981-493d-9ee7-a6dad6887d64 + responseCode: OK + result: + data: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry_record-t4 + name: sb-telemetry + type: event + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_events + name: sb-telemetry + type: event + status: Draft + tags: [] + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_record-master + name: sb-telemetry + type: master + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: generate-schema + name: generate-schema + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-test + name: trip-test + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample-trip-details + name: sample-trip-details + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: {} + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: {} + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 30 + example-1: + summary: 'Success: Filter based on status as array' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T17:57:38+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 31aba5bc-8492-45ce-be0e-8c52d8716014 + responseCode: OK + result: + data: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + count: 16 + example-2: + summary: 'Success: Filter basen on status as string' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T17:59:18+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a08c7ea0-bb1c-4998-b47d-a76e38e87e31 + responseCode: OK + result: + data: + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: {} + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: {} + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 8 + example-3: + summary: 'Success: Filter based on dataset type' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T18:00:41+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 0d1ff2de-42c9-4192-b75d-84f711dbfb55 + responseCode: OK + result: + data: + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + count: 2 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T18:02:26+05:30' + params: + status: FAILED + resmsgid: add9dbe0-f362-4f99-890c-3387c998a049 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_LIST_INPUT_INVALID + message: >- + #properties/params/required must have required property + 'msgid' + /v2/datasets/dataschema: + post: + tags: + - Dataset api's + summary: Data schema generator + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + data: + - eid: IMPRESSION + ets: 1672657002221 + ver: '3.0' + mid: IMPRESSION:2b5834e196f485c17c4e49d292af43c0 + actor: + id: 0c45959486f579c24854d40a225d6161 + type: User + context: + channel: '01268904781886259221' + pdata: + id: staging.diksha.portal + ver: 5.1.0 + pid: sunbird-portal + env: public + sid: 23850c90-8a8c-11ed-95d0-276800e1048c + did: 0c45959486f579c24854d40a225d6161 + cdata: [] + rollup: + l1: '01268904781886259221' + uid: anonymous + object: {} + tags: + - '01268904781886259221' + edata: + type: view + pageid: login + subtype: pageexit + uri: >- + https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4 + visits: [] + syncts: 1672657005814 + '@timestamp': '2023-01-02T10:56:45.814Z' + flags: + ex_processed: true + - eid: IMPRESSION + ets: 1672656997928 + ver: '3.0' + mid: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: POST + - url: /v1/org/search + - duration: 0 + - status: OK + - eid: LOG + ets: 1672656998024 + ver: '3.0' + mid: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + type=system, message=EXIT LOG: method : POST, url: + /v1/org/search , For Operation : orgSearch, + params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + errmsg=Invalid value null for parameter hashTagId. + Please provide a valid value., + resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + err=UOS_ORGSER0017, status=FAILED, + responseCode=400}]}} + params: [] + - eid: LOG + ets: 1672657004961 + ver: '3.0' + mid: f34112c7242a3e3a26f0015796b029c2 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: f34112c7242a3e3a26f0015796b029c2 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + ElasticSearchRestHighImpl:search: calling search for + index org_alias, with query = + {"from":0,"size":250,"query":{"bool":{"must":[{"term":{"isTenant.raw":{"value":true,"boost":1.0}}},{"term":{"slug.raw":{"value":"ntp","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":[],"excludes":[]}} + params: [] + - eid: LOG + ets: 1672657006595 + ver: '3.0' + mid: d23ff123-40f0-4262-a69b-b75b46d315a1 + actor: + id: 930a3994-cbe7-4e84-936f-4974096af6f2 + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: d23ff123-40f0-4262-a69b-b75b46d315a1 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, + type=system, message=ENTRY LOG: method : GET, url: + /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 + , For Operation : getUserRolesById, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006611 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, + type=system, message=ENTRY LOG: method : GET, url: + /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , + For Operation : getUserProfileV5, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006620 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user_roles + WHERE userId=?; + params: [] + - eid: LOG + ets: 1672657006645 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: GET + - url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 + - duration: 0 + - status: OK + - eid: LOG + ets: 1672657007238 + ver: '3.0' + mid: d4d34fde-c407-efb6-03bd-9f892ca0f114 + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.portal + pid: learner-service + ver: 5.0.0 + env: User + did: d904c90d9f81ddac20141b94ddd606a0 + cdata: + - id: d4d34fde-c407-efb6-03bd-9f892ca0f114 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user WHERE + id=?; + params: [] + config: + dataset: financial_transactions + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v1 + ts: '2024-07-22T12:32:50+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1309aea0-9a97-46e9-bc5e-a16a8a7fb624 + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' format + type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'sid'. The property sid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.sid + - message: >- + The Property 'context.sid' appears to be + 'uuid' format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.context.properties.sid + arrival_format: text + data_type: string + did: + type: string + arrival_format: text + data_type: string + cdata: + type: array + items: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'context.cdata[*].id' + appears to be 'uuid' format type. + advice: >- + Suggest to not to index the high + cardinal columns + resolutionType: DEDUP + severity: LOW + path: >- + properties.context.properties.cdata.items.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: array + data_type: array + rollup: + type: object + properties: + l1: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at + property: 'l1'. The property l1: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System + has updated the property schema to + optional + resolutionType: OPTIONAL + severity: MEDIUM + path: >- + properties.context.properties.rollup.properties.l1 + arrival_format: text + data_type: string + arrival_format: object + data_type: object + uid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uid'. The property uid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.uid + arrival_format: text + data_type: string + arrival_format: object + data_type: object + object: + type: object + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'object'. The property object: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.object + arrival_format: object + data_type: object + tags: + type: array + items: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'tags'. The property tags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.tags + arrival_format: array + data_type: array + edata: + type: object + properties: + type: + type: string + arrival_format: text + data_type: string + pageid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'pageid'. The property pageid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.pageid + arrival_format: text + data_type: string + subtype: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'subtype'. The property subtype: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.subtype + arrival_format: text + data_type: string + uri: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uri'. The property uri: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.uri + - message: >- + The Property 'edata.uri' appears to be 'uri' + format type. + severity: '' + path: properties.edata.properties.uri + arrival_format: text + data_type: string + visits: + type: array + items: {} + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'visits'. The property visits: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.visits + arrival_format: array + data_type: array + level: + type: string + arrival_format: text + data_type: string + message: + type: string + arrival_format: text + data_type: string + params: + type: array + items: {} + arrival_format: array + data_type: array + arrival_format: object + data_type: object + syncts: + type: integer + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'syncts'. The property syncts: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.syncts + - message: >- + The Property 'syncts' appears to be 'epoch' format + type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + '@timestamp': + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + '@timestamp'. The property @timestamp: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.@timestamp + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + flags: + type: object + properties: + ex_processed: + type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'ex_processed'. The property ex_processed: + only 1 time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags.properties.ex_processed + arrival_format: boolean + data_type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'flags'. The property flags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + mid: + path: $.mid + cardinality: 67 + index: false + actor.id: + path: $.actor.properties.id + cardinality: 56 + index: false + context.sid: + path: $.context.properties.sid + cardinality: 11 + index: true + edata.uri: + path: $.edata.properties.uri + cardinality: 11 + index: true + context.cdata[*].id: + path: $.context.properties.cdata.items.properties.id + cardinality: 62 + index: false + processing: + dedupKeys: + - mid + - context.cdata[*].id + - actor.id + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request body' + value: + id: api.datasets.dataschema + ver: v1 + ts: '2024-07-22T12:33:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bbcc86c2-042d-4f77-bb6e-e1c9116df570 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'data' + example-1: + summary: 'Failure: Invalid request (config not provided)' + value: + id: api.datasets.dataschema + ver: v1 + ts: '2024-07-22T12:35:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1f856c5e-37f0-41e9-96fb-642471228da2 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'config' + /v2/files/generate-url: + post: + tags: + - Dataset api's + summary: File generate url + requestBody: + content: + application/json: + examples: + example1: + summary: Request body for put url + value: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + example2: + summary: Request body for get url + value: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: read + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' + /v2/connectors/list: + post: + tags: + - Connector api's + summary: Connector list + requestBody: + content: + application/json: + examples: + example1: + summary: Filter based on category + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + category: + - Database + example2: + summary: Connectors list with all filter options + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + category: + - Database, File + status: + - Live, Draft, InValidation, Retired + example3: + summary: Connectors list without filters + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: {} + example4: + summary: Filtered based on status + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Draft + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Filtered based on status' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:25:51+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a + responseCode: OK + result: + data: [] + count: 0 + example-1: + summary: 'Success: Filtered based on category' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-31T18:55:03+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + count: 4 + example-2: + summary: 'Success: Connectors list with all filter options' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-31T18:56:32+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 11a2f537-bd98-405b-97e5-0f0d5b86b2c3 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-3: + summary: 'Success: Connectors list without filters' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-31T18:57:37+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c2467e01-0a2d-401c-aa3d-dd16b804f723 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:13:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fadde0-8c59-4420-8ab3-56474b01670b + responseCode: BAD_REQUEST + result: {} + error: + code: CONNECTORS_LIST_INPUT_INVALID + message: >- + #properties/request/properties/filters/properties/status/minItems + must NOT have fewer than 1 items + /v2/connectors/read/postgres-connector-1.0.0: + get: + tags: + - Connector api's + summary: Connector Read + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Read live Connector' + value: + id: api.connectors.read + ver: v2 + ts: '2024-07-31T18:17:54+05:30' + params: + status: SUCCESS + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced + responseCode: OK + result: + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: + type: object + properties: + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: + type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly + fieldDescription: + - type: string + description: '' + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: + batch_size: + ui:readonly: true + max_batches: + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + example-1: + summary: 'Success: Read Draft Connector' + value: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T12:47:12+05:30' + params: + status: SUCCESS + resmsgid: b6fcfb05-246c-4a1b-9eb1-27497ee9b80b + responseCode: OK + result: + id: mssql-connector-2.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 2.0.0 + description: >- + The MS SQL Connector is used to move data from any MS + SQL Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Draft + ui_spec: {} + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T15:02:48+05:30' + params: + status: FAILED + resmsgid: 712e7298-99f8-4694-9011-4232fcfd664a + responseCode: NOT_FOUND + result: {} + error: + code: CONNECTOR_NOT_FOUND + message: 'Connector not found: postgres-conn' \ No newline at end of file diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 4d0d51f3..2ca4edef 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -1,11 +1,22 @@ -FROM --platform=linux/amd64 python:3.10-alpine +FROM --platform=linux/amd64 python:3.12-alpine COPY --from=ubuntu /usr/local/bin /usr/local/bin -RUN apk update && apk add curl jq && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ -RUN apk add libcrypto3=3.1.4-r5 -RUN apk upgrade + +RUN apk update && apk add curl jq vim && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ + +RUN cd /tmp && curl -OL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ + && tar -zxvf helm-v3.13.2-linux-amd64.tar.gz \ + && mv linux-amd64/helm /usr/local/bin/helm + +# RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ +# && mv kubectl /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl + +# RUN apk add openssl==3.3.0-r2 libcrypto3==3.3.0-r2 || \ +RUN apk add openssl libcrypto3 +RUN apk upgrade --no-cache WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY src ./src +COPY helm-charts ./helm-charts WORKDIR /app/src CMD [ "uvicorn", "routes:app", "--host", "0.0.0.0" ] \ No newline at end of file diff --git a/command-service/README.md b/command-service/README.md index 5c06533a..d11413b7 100644 --- a/command-service/README.md +++ b/command-service/README.md @@ -3,12 +3,13 @@ ### Commands * Each command implementation under the command module will extend from icommand interface and will have to implement the execute function. -* The execute function will take a command payload json object and also an action as input. -* Currently implemented FlinkCommand class. +* The execute function will take a command payload json object and also an action as input. For e.g. The DruidCommand class' execute function will take the command payload and SUBMIT_INGESTION_TASK action as two parameters. +* Currently implemented command classes are DruidCommand, FlinkCommand and DbCommand classes. ### Services -Currently, there are one generic service under the services module: +Currently, there are two generic services under the services module: +* DatabaseService implements all the required database operations such as select_one, select_all and upsert operations from Postgresql. The service uses psycopg2 library to connect to Postgres. * HttpService implements GET, POST and DELETE operations. The service uses urllib3 library to invoke http urls. ### Configuration @@ -16,7 +17,7 @@ Currently, there are one generic service under the services module: * The config.py class has an utility implementation `find` to read nested configurations from a yaml file. * The service_config.yml class has all the required configurations for the service. - The flink.jobs configuration is required to specify the list of jobs and the corresponding job_manager_urls. This is required for restarting the required jobs. - - The commands entry will have the workflow of sub-commands for each higher level comamnd. For e.g., RESTART_PIPELINE command is comprised for RESTART_PIPELINE_JOBS sub-command. + - The commands entry will have the workflow of sub-commands for each higher level comamnd. For e.g., PUBLISH_DATASET command is comprised for five sub-commands such as MAKE_DATASET_LIVE, SUBMIT_INGESTION_TASKS, STOP_PIPELINE_JOBS and START_PIPELINE_JOBS. ### Deployment diff --git a/command-service/docs/pii_swagger_doc.yml b/command-service/docs/pii_swagger_doc.yml new file mode 100644 index 00000000..f0992894 --- /dev/null +++ b/command-service/docs/pii_swagger_doc.yml @@ -0,0 +1,147 @@ +openapi: 3.0.0 +info: + title: PII Detection API + description: A simple API for detecting PII fields in dataset sample. + version: 1.0.0 +servers: + - url: http://{url}:{port}/data/v1 +paths: + /analyze/pii: + post: + summary: return fields with PII data flagged with explanation and confidence + operationId: detectPII + requestBody: + description: dataset id and sample single flattened event with all fields populated + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PIIRequest' + responses: + '200': + description: A list of potential fields with PII + content: + application/json: + schema: + $ref: '#/components/schemas/PIIResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/PIIFailedResponse' +components: + schemas: + PIIRequest: + type: object + properties: + dataset_id: + type: string + example: telemetry-data + data: + type: array + items: + type: object + required: + - dataset_id + - data + PIIResponse: + type: object + properties: + id: + type: string + example: dc235b34-32b1-4d5e-b1c8-985f5e4d30d9 + response_code: + type: string + example: OK + status_code: + type: integer + example: 200 + result: + type: array + items: + type: object + properties: + field: + type: string + description: the flattened column name/path + example: user.mobile + type: + type: string + description: whether the field is detected as a name/phone/email/ID/location + example: phone + score: + type: number + description: a 0-100% value signifying the confidence in the detection + example: 0.75 + reason: + type: array + items: + properties: + code: + type: string + description: reason code for i18n code + example: <> + resourceKey:: + type: string + description: resource bundle id for UI to display proper label + example: pii.descriptions.m001 + region: + type: string + description: Optional. whether related to any specific region + example: IN + score: + type: number + description: Confidence score specific to region + example: 0.75 + required: + - code + - message + ts: + type: string + example: 1705645119000 + params: + type: object + properties: + status: + type: string + required: + - id + - response_code + - status_code + PIIFailedResponse: + type: object + properties: + id: + type: string + example: dc235b34-32b1-4d5e-b1c8-985f5e4d30d9 + response_code: + type: string + example: INTERNAL_SERVER_ERROR + status_code: + type: integer + example: 500 + result: + type: object + properties: + errorCode: + type: integer + example: 500 + errorMsg: + type: string + example: Error parsing the sample event + errorTrace: + type: string + example: "SyntaxError: JSON.parse: bad character in string literal" + ts: + type: string + example: 1705645119000 + params: + type: object + properties: + status: + type: string + required: + - id + - response_code + - status_code diff --git a/command-service/helm-charts/connector-cron-jobs/Chart.yaml b/command-service/helm-charts/connector-cron-jobs/Chart.yaml new file mode 100644 index 00000000..81eb2cf0 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: connector-cron-jobs +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl b/command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl new file mode 100644 index 00000000..84c16281 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl @@ -0,0 +1,74 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "connector-cron-jobs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "connector-cron-jobs.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "connector-cron-jobs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "connector-cron-jobs.labels" -}} +helm.sh/chart: {{ include "connector-cron-jobs.chart" . }} +{{ include "connector-cron-jobs.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "connector-cron-jobs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "connector-cron-jobs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +# .Values.namespace will get overridden by .Values.global.namespace.chart-name +{{- define "base.namespace" -}} + {{- $chartName := .Chart.Name }} + {{- $namespace := default .Release.Namespace .Values.namespace }} + {{- if .Values.global }} + {{- with .Values.global.namespace }} + {{- if hasKey . $chartName }} + {{- $namespace = index . $chartName }} + {{- end }} + {{- end }} + {{- end }} + {{- $namespace | trunc 63 | trimSuffix "-" }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "base.serviceaccountname" -}} + {{- $name := printf "%s-%s" .Chart.Name "sa" }} + {{- default $name .Values.serviceAccount.name }} +{{- end }} diff --git a/command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml b/command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml new file mode 100644 index 00000000..a0611094 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml @@ -0,0 +1,59 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: "{{ .Values.instance-id }}-cronjob" +spec: + schedule: "{{ .Values.schedule }}" + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + containers: + - name: spark-submit-container + image: bitnami/kubectl:latest + pullPolicy: IfNotPresent + command: + {{- if eq .Values.technology "scala" }} + - /bin/sh + - -c + - | + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --jars /data/connectors/{{ .Values.connector-source }}/libs/\*.jar --class {{ .Values.main_class }} /data/connectors/{{ .Values.connector-source }}/{{ .Values.main_file }} -c {{ .Values.instance-id }}" + {{- else if eq .Values.technology "python" }} + - /bin/sh + - -c + - | + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector-source }}/libs/\* /data/connectors/{{ .Values.connector-source }}/{{ .Values.main_file }} -c {{ .Values.instance-id }}" + {{- end }} + + # - name: connector-cron-jobs + # image: alpine/curl + # command: ["/bin/sh", "-c"] + # args: + # - > + # CURRENT_TIME=$(date +%Y-%m-%dT%H:%M:%S.%sZ) && curl -vi --location + # "{{ .Values.spark.host }}" + # --header "Content-Type: application/json" + # --data + # '{ + # "file": "{{ .Values.file.path }}", + # "args": ["{{ .Values.args }}"], + # "name": "{{ .Values.job.name }}-'"$CURRENT_TIME"'", + # "className": "{{ .Values.class.name }}", + # "executorCores": 1, + # "executorMemory": "1G", + # "numExecutors": 1, + # "conf": { + # "spark.driver.extraJavaOptions": "{{ .Values.spark.driver.extraJavaOptions }}", + # "spark.executor.extraJavaOptions": "{{ .Values.spark.executor.extraJavaOptions }}", + # "spark.master": "spark://{{ .Values.spark.master.host }}:{{ .Values.spark.master.port }}" + # } + # }' diff --git a/command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml b/command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml new file mode 100644 index 00000000..7e291fd6 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml @@ -0,0 +1,48 @@ +{{- if .Values.rbac.enabled -}} +--- +{{- if .Values.rbac.useClusterRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "common.names.fullname" . }} +rules: +{{- toYaml .Values.rbac.rules | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "common.names.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "common.names.fullname" . }} +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} +rules: +{{- toYaml .Values.rbac.rules | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} +subjects: +- kind: ServiceAccount + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "common.names.fullname" . }} +{{- end }} + +{{- end }} + diff --git a/command-service/helm-charts/connector-cron-jobs/values.yaml b/command-service/helm-charts/connector-cron-jobs/values.yaml new file mode 100644 index 00000000..15127206 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/values.yaml @@ -0,0 +1,29 @@ +## JDBC Connector +technology: scala +connector-source: jdbc-connector-1.0.0 +instance-id: nyt-psql.1 +main_class: org.sunbird.obsrv.connector.JDBCConnector +main_file: jdbc-connector-1.0.0.jar + + +## Object Store Connector +# technology: python +# connector-source: object_store_connector-0.1.0 +# instance-id: s3.new-york-taxi-data.1 +# main_file: object_store_connector/__main__.py + +schedule: 0 * * * * # Every hour + +## Defaults + +python_path: /opt/bitnami/python/bin/python + +spark: + host: http://spark-master-svc.spark.svc.cluster.local:8998/batches/ + master: + host: spark-master-svc.spark.svc.cluster.local + port: 7077 + driver: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties + executor: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties diff --git a/command-service/helm-charts/flink-connector/Chart.lock b/command-service/helm-charts/flink-connector/Chart.lock new file mode 100644 index 00000000..fc1e9b7d --- /dev/null +++ b/command-service/helm-charts/flink-connector/Chart.lock @@ -0,0 +1,18 @@ +dependencies: +- name: common + repository: https://nimbushubin.github.io/helmcharts + version: 0.1.0 +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 18.1.1 +- name: kafka + repository: https://charts.bitnami.com/bitnami + version: 20.0.2 +- name: cassandra + repository: https://charts.bitnami.com/bitnami + version: 10.1.0 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 12.2.7 +digest: sha256:7ebab454bd79d34678de3d11bcc4b10c25f0821d693e7a7399e4c8bebe742313 +generated: "2023-12-12T13:15:34.600338+05:30" diff --git a/command-service/helm-charts/flink-connector/Chart.yaml b/command-service/helm-charts/flink-connector/Chart.yaml new file mode 100644 index 00000000..7b2891b5 --- /dev/null +++ b/command-service/helm-charts/flink-connector/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +appVersion: 1.0.0 +description: A production-ready Helm chart base template +maintainers: + - name: NimbusHub.in +name: flink +version: 0.1.0 diff --git a/command-service/helm-charts/flink-connector/charts/.helmignore b/command-service/helm-charts/flink-connector/charts/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/command-service/helm-charts/flink-connector/charts/common/Chart.yaml b/command-service/helm-charts/flink-connector/charts/common/Chart.yaml new file mode 100644 index 00000000..cc1e9676 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +description: Library chart for Sunbird +keywords: +- common +- helper +- template +- function +maintainers: +- name: NimbusHub.in +name: common +type: library +version: 0.1.0 diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..e85b1df4 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl @@ -0,0 +1,139 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a topologyKey definition +{{ include "common.affinities.topologyKey" (dict "topologyKey" "BAR") -}} +*/}} +{{- define "common.affinities.topologyKey" -}} +{{ .topologyKey | default "kubernetes.io/hostname" -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: 1 + {{- range $extraPodAffinityTerms }} + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: {{ .weight | default 1 -}} + {{- end -}} +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- range $extraPodAffinityTerms }} + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- end -}} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..b1257397 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl @@ -0,0 +1,229 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for daemonset. +*/}} +{{- define "common.capabilities.daemonset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Vertical Pod Autoscaler. +*/}} +{{- define "common.capabilities.vpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if PodSecurityPolicy is supported +*/}} +{{- define "common.capabilities.psp.supported" -}} +{{- if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if AdmissionConfiguration is supported +*/}} +{{- define "common.capabilities.admisionConfiguration.supported" -}} +{{- if semverCompare ">=1.23-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for AdmissionConfiguration. +*/}} +{{- define "common.capabilities.admisionConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiserver.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodSecurityConfiguration. +*/}} +{{- define "common.capabilities.podSecurityConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "pod-security.admission.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..07ded6f6 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl new file mode 100644 index 00000000..1bcb779d --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl @@ -0,0 +1,117 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" .Values.global ) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $separator := ":" -}} +{{- $termination := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .imageRoot.digest }} + {{- $separator = "@" -}} + {{- $termination = .imageRoot.digest | toString -}} +{{- end -}} +{{- if $registryName }} + {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}} +{{- else -}} + {{- printf "%s%s%s" $repositoryName $separator $termination -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end }} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper image version (ingores image revision/prerelease info & fallbacks to chart appVersion) +{{ include "common.images.version" ( dict "imageRoot" .Values.path.to.the.image "chart" .Chart ) }} +*/}} +{{- define "common.images.version" -}} +{{- $imageTag := .imageRoot.tag | toString -}} +{{/* regexp from https://github.com/Masterminds/semver/blob/23f51de38a0866c5ef0bfc42b3f735c73107b700/version.go#L41-L44 */}} +{{- if regexMatch `^([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?$` $imageTag -}} + {{- $version := semver $imageTag -}} + {{- printf "%d.%d.%d" $version.Major $version.Minor $version.Patch -}} +{{- else -}} + {{- print .chart.AppVersion -}} +{{- end -}} +{{- end -}} + diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..efa5b85c --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl @@ -0,0 +1,73 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") (hasKey .annotations "kubernetes.io/tls-acme") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..d90a6cdc --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl @@ -0,0 +1,46 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Kubernetes standard labels +{{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "common.labels.standard" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{- $default := dict "app.kubernetes.io/name" (include "common.names.name" .context) "helm.sh/chart" (include "common.names.chart" .context) "app.kubernetes.io/instance" .context.Release.Name "app.kubernetes.io/managed-by" .context.Release.Service -}} +{{- with .context.Chart.AppVersion -}} +{{- $_ := set $default "app.kubernetes.io/version" . -}} +{{- end -}} +{{ template "common.tplvalues.merge" (dict "values" (list .customLabels $default) "context" .context) }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Labels used on immutable fields such as deploy.spec.selector.matchLabels or svc.spec.selector +{{ include "common.labels.matchLabels" (dict "customLabels" .Values.podLabels "context" $) -}} + +We don't want to loop over custom labels appending them to the selector +since it's very likely that it will break deployments, services, etc. +However, it's important to overwrite the standard labels if the user +overwrote them on metadata.labels fields. +*/}} +{{- define "common.labels.matchLabels" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{ merge (pick (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) "app.kubernetes.io/name" "app.kubernetes.io/instance") (dict "app.kubernetes.io/name" (include "common.names.name" .context) "app.kubernetes.io/instance" .context.Release.Name ) | toYaml }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl new file mode 100644 index 00000000..a222924f --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl @@ -0,0 +1,71 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a193c46b --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl @@ -0,0 +1,172 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + - failOnNew - Boolean - Optional - Default to true. If set to false, skip errors adding new keys to existing secrets. +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $failOnNew := default true .failOnNew }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key | quote }} + {{- else if $failOnNew }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Reuses the value from an existing secret, otherwise sets its value to a default value. + +Usage: +{{ include "common.secrets.lookup" (dict "secret" "secret-name" "key" "keyName" "defaultValue" .Values.myValue "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - defaultValue - String - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - context - Context - Required - Parent context. + +*/}} +{{- define "common.secrets.lookup" -}} +{{- $value := "" -}} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data -}} +{{- if and $secretData (hasKey $secretData .key) -}} + {{- $value = index $secretData .key -}} +{{- else if .defaultValue -}} + {{- $value = .defaultValue | toString | b64enc -}} +{{- end -}} +{{- if $value -}} +{{- printf "%s" $value -}} +{{- end -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..16405a0f --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..a8ed7637 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,38 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + +{{/* +Merge a list of values that contains template after rendering them. +Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge +Usage: +{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }} +*/}} +{{- define "common.tplvalues.merge" -}} +{{- $dst := dict -}} +{{- range .values -}} +{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}} +{{- end -}} +{{ $dst | toYaml }} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..bfbddf05 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ include "common.names.namespace" .context | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Checksum a template at "path" containing a *single* resource (ConfigMap,Secret) for use in pod annotations, excluding the metadata (see #18376). +Usage: +{{ include "common.utils.checksumTemplate" (dict "path" "/configmap.yaml" "context" $) }} +*/}} +{{- define "common.utils.checksumTemplate" -}} +{{- $obj := include (print .context.Template.BasePath .path) .context | fromYaml -}} +{{ omit $obj "apiVersion" "kind" "metadata" | toYaml | sha256sum }} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl new file mode 100644 index 00000000..e5572991 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl @@ -0,0 +1,41 @@ +{{/* Define findkey function */}} +{{- define "common.variables.findkey" -}} +{{- $value := .value -}} +{{- $keys := .keys -}} +{{- $found := true -}} + +{{- range $key := $keys -}} + {{- if kindIs "map" $value -}} + {{- if hasKey $value $key -}} + {{- $value = index $value $key -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} +{{- end -}} + +{{- if $found -}} + {{- $value -}} +{{- else -}} + {{- "" -}} +{{- end -}} +{{- end -}} + +{{/* Define getmethekey function with precedence and input as a dot-delimited string using findkey */}} +{{- define "common.variables.variableGlobal" -}} +{{- $root := first . -}} +{{- $keyString := index (rest .) 0 -}} +{{- $keys := splitList "." $keyString -}} + +{{- $value := include "common.variables.findkey" (dict "value" $root.Values "keys" $keys) -}} + +{{- if (not $value) -}} + {{- $value = include "common.variables.findkey" (dict "value" $root.Values.global "keys" $keys) -}} +{{- end -}} + +{{- $value -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..66dffc1f --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl @@ -0,0 +1,19 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..eda9aada --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..17d83a2f --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..bbb445b8 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,113 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..ca3953f8 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..8c9aa570 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,134 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..fc0d208d --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,81 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..31ceda87 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,51 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink-connector/charts/common/values.yaml b/command-service/helm-charts/flink-connector/charts/common/values.yaml new file mode 100644 index 00000000..c83b33f2 --- /dev/null +++ b/command-service/helm-charts/flink-connector/charts/common/values.yaml @@ -0,0 +1,82 @@ +# Default values for common. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/command-service/helm-charts/flink-connector/templates/NOTES.txt b/command-service/helm-charts/flink-connector/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl b/command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl new file mode 100644 index 00000000..dadfc22d --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl @@ -0,0 +1,4 @@ +{{- define "base.serviceaccountname" -}} + {{- $name := include "common.names.fullname" . }} + {{- default $name .Values.serviceAccount.name }} +{{- end }} diff --git a/command-service/helm-charts/flink-connector/templates/_helpers.tpl b/command-service/helm-charts/flink-connector/templates/_helpers.tpl new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/flink-connector/templates/_image_flink.tpl b/command-service/helm-charts/flink-connector/templates/_image_flink.tpl new file mode 100644 index 00000000..5c542f64 --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/_image_flink.tpl @@ -0,0 +1,16 @@ +{{/* {{- include "base.image.flink" dict ("context" $ "scope" $jobData) }} */}} +{{- define "base.image.flink" }} +{{- $context := .context }} +{{- $scope := .scope }} +{{- with $scope }} +{{- $registry := default $context.Values.registry .registry }} +{{- $repository := default $context.Values.repository .repository }} +{{- $image := printf "%s/%s" $registry $repository}} +{{- if .digest }} +{{- printf "%s@%s" $image .digest }} +{{- else }} +{{- $tag := default $context.Values.tag .tag }} +{{- printf "%s:%s" $image $tag }} +{{- end }} +{{- end }} +{{- end }} diff --git a/command-service/helm-charts/flink-connector/templates/_namespace.tpl b/command-service/helm-charts/flink-connector/templates/_namespace.tpl new file mode 100644 index 00000000..7af0b70b --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/_namespace.tpl @@ -0,0 +1,13 @@ +# .Values.namespace will get overridden by .Values.global.namespace.chart-name +{{- define "base.namespace" -}} + {{- $chartName := .Chart.Name }} + {{- $namespace := default .Release.Namespace .Values.namespace }} + {{- if .Values.global }} + {{- with .Values.global.namespace }} + {{- if hasKey . $chartName }} + {{- $namespace = index . $chartName }} + {{- end }} + {{- end }} + {{- end }} + {{- $namespace | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/command-service/helm-charts/flink-connector/templates/configmap.yaml b/command-service/helm-charts/flink-connector/templates/configmap.yaml new file mode 100644 index 00000000..3a90fa01 --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/configmap.yaml @@ -0,0 +1,21 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- with $currentScope }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-config" $jobName }} + namespace: {{ include "base.namespace" . }} + labels: {{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: +{{ include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +data: + {{- "log4j_console_properties" | nindent 2 -}}: |- + {{- include "common.tplvalues.render" (dict "value" .Values.log4j_console_properties "context" $) | nindent 4 }} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml new file mode 100644 index 00000000..74fc9cca --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -0,0 +1,238 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- with $currentScope }} +{{ $component := "taskmanager" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-%s" $jobName $component }} + namespace: {{ include "base.namespace" $ }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + {{- if .Values.commonAnnotations }} + {{ include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + component: {{ printf "%s-%s" $jobName $component }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + spec: + {{- if .Values.serviceAccount.create}} + serviceAccountName: {{ include "base.serviceaccountname" . }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- $imagePullSecrets := default .Values.imagePullSecrets $jobData.imagePullSecrets }} + {{- with $imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $jobName }}-taskmanager + image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} + workingDir: {{ .Values.taskmanager.flink_work_dir }} + args: ["taskmanager"] + env: + - name: FLINK_PROPERTIES + value: |+ + jobmanager.rpc.address: {{ $jobName }}-jobmanager + taskmanager.rpc.port=6122 + taskmanager.numberOfTaskSlots: 2 + metrics.reporters: prom + metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + metrics.reporter.prom.host: {{ $jobName }}-taskmanager + metrics.reporter.prom.port: 9251 + ports: + - containerPort: {{ .Values.taskmanager.rpc_port }} + name: rpc + {{- toYaml (index .Values.flink_resources $component )| nindent 10 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + - name: flink-common-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + volumes: + - name: flink-common-volume + configMap: + name: {{ $jobName }}-config + items: + - key: log4j_console_properties + path: log4j-console.properties +{{ $component := "jobmanager" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ printf "%s-%s" $jobName $component }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + {{- if .Values.commonAnnotations }} + {{ include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + component: {{ printf "%s-%s" $jobName $component }} + annotations: + checksum/config: {{ .Files.Glob "cFonfigs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + spec: + serviceAccountName: {{ include "base.serviceaccountname" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- $imagePullSecrets := default .Values.imagePullSecrets $jobData.imagePullSecrets }} + {{- with $imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $jobName }}-jobmanager + image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} + workingDir: /opt/flink + # command: ["/bin/sh", "-c"] + args: ["jobmanager"] + env: + - name: FLINK_PROPERTIES + value: |+ + jobmanager.rpc.address: {{ $jobName }}-jobmanager + jobmanager.rpc.port=6123 + metrics.reporters: prom + metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + metrics.reporter.prom.host: {{ $jobName }}-jobmanager + metrics.reporter.prom.port: 9250 + volumeMounts: + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-scala-config.conf + subPath: connectors-scala-config.conf + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml + - name: data + mountPath: /data/connectors + - name: flink-common-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + + - name: {{ $jobName }}-job-submit + image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} + workingDir: /opt/flink + env: + - name: CONNECTOR_ID + value: {{ index $jobData "connector_id" }} + - name: CONFIG_PATH + value: /data/flink/conf/ + - name: CONFIG_FILE + value: connectors-python-config.yaml + command: + - /bin/bash + - -c + - | + /usr/bin/python3 /data/connectors-init/connector.py; + sleep 30s; + /opt/flink/bin/flink run -d -m \ + {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ + /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ + --config.file.path /data/flink/conf/connectors-scala-config.conf \ + --metadata.id {{ index $jobData "connector_id" }} \ + {{- if eq .Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ + {{- end }} + {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} + "-Ds3.access-key={{ .Values.s3_access_key }}" \ + "-Ds3.secret-key={{ .Values.s3_secret_key }}" \ + "-Ds3.endpoint={{ .Values.s3_endpoint }}" \ + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}" \ + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcs" }} + "-Dgoogle.cloud.auth.service.account.enable=true" \ + {{- end }} + ; sleep 10s; echo "Job submitted"; tail -f /dev/null; + ports: + {{- range .Values.service.ports }} + - name: {{ .name }} + containerPort: {{ .targetPort }} + {{- end }} + {{- toYaml (index .Values.flink_resources $component )| nindent 10 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-scala-config.conf + subPath: connectors-scala-config.conf + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml + - name: data + mountPath: /data/connectors + - name: flink-common-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + volumes: + - name: connector-config-volume + configMap: + name: connectors-config + items: + - key: connectors-scala-config.conf + path: connectors-scala-config.conf + - key: connectors-python-config.yaml + path: connectors-python-config.yaml + - name: flink-common-volume + configMap: + name: {{ $jobName }}-config + items: + # - key: flink-conf + # path: flink-conf.yaml + - key: log4j_console_properties + path: log4j-console.properties + + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + +{{- end }} +{{- end}} +{{- end}} \ No newline at end of file diff --git a/command-service/helm-charts/flink-connector/templates/hpa.yaml b/command-service/helm-charts/flink-connector/templates/hpa.yaml new file mode 100644 index 00000000..a9600ffb --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/hpa.yaml @@ -0,0 +1,20 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "common.names.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- toYaml .Values.autoscaling.metrics | nindent 4 }} +{{- end }} diff --git a/command-service/helm-charts/flink-connector/templates/ingress.yaml b/command-service/helm-charts/flink-connector/templates/ingress.yaml new file mode 100644 index 00000000..7a01585c --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/ingress.yaml @@ -0,0 +1,24 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} + annotations: + {{- toYaml .Values.ingress.annotations | nindent 4 }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + service: + name: {{ include "common.names.fullname" $ }} + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/command-service/helm-charts/flink-connector/templates/service.yaml b/command-service/helm-charts/flink-connector/templates/service.yaml new file mode 100644 index 00000000..e1c3f711 --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/service.yaml @@ -0,0 +1,42 @@ +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- $components := list "jobmanager" "taskmanager" }} +{{- range $component := $components }} +--- +apiVersion: v1 +kind: Service +metadata: + namespace: {{ include "base.namespace" $}} + name: {{ $jobName }}-{{ $component }} + labels: + {{- include "common.labels.standard" (dict "customLabels" $.Values.commonLabels "context" $) | nindent 4 }} + component: {{ printf "%s-%s" $jobName $component }} + + {{- if $.Values.commonAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" $.Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + {{- if eq $component "jobmanager" }} + - name: rpc + port: {{ $.Values.jobmanager.rpc_port }} + - name: blob + port: {{ $.Values.jobmanager.blob_port }} + - name: query + port: {{ $.Values.jobmanager.query_port }} + - name: ui + port: {{ $.Values.jobmanager.ui_port }} + - name: prom + port: {{ $.Values.jobmanager.prom_port }} + {{- else if eq $component "taskmanager" }} + - name: prom + port: {{ $.Values.taskmanager.prom_port }} + {{- end }} + selector: + app.kubernetes.io/component: {{ $jobName }}-{{ $component }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/flink-connector/templates/serviceaccount.yaml b/command-service/helm-charts/flink-connector/templates/serviceaccount.yaml new file mode 100644 index 00000000..d26e8536 --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} + {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/command-service/helm-charts/flink-connector/templates/servicemonitor.yaml b/command-service/helm-charts/flink-connector/templates/servicemonitor.yaml new file mode 100644 index 00000000..e934bda0 --- /dev/null +++ b/command-service/helm-charts/flink-connector/templates/servicemonitor.yaml @@ -0,0 +1,31 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled}} +{{- range $key, $smData := $.Values.serviceMonitor }} +{{- if $smData.enabled}} +{{- with $currentScope }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ printf "%s-%s" $jobName $key }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ $smData.jobLabel }} + selector: + matchLabels: + {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 6 }} + endpoints: + - port: {{ $smData.port }} # the name of the port in your service, assuming the primary service port is named 'http' in this example. + interval: {{ $smData.interval }} + scrapeTimeout: {{ $smData.scrapeTimeout }} + honorLabels: {{ $smData.honorLabels }} +{{- end}} +{{- end}} +{{- end}} +{{- end}} +{{- end}} \ No newline at end of file diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml new file mode 100644 index 00000000..dd08dafa --- /dev/null +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -0,0 +1,253 @@ +# vim: set fdm=indent: +nameOverride: "" +fullnameOverride: "" + +replicaCount: 1 + +namespace: "flink" +commonLabels: + system.processing: "true" + release: monitoring + app: flink + + +# repository: sunbirded.azurecr.io/data-pipeline +# tag: "release-5.2.0_RC1_2c615f8_12" +# docker pull sunbirded.azurecr.io/sunbird-datapipeline:release-4.9.0_RC4_1 +registry: sanketikahub +repository: flink-connectors +tag: 1.17.2-scala_2.12-java11 +imagePullSecrets: [] +imagePullPolicy: IfNotPresent + +## Databases +global: + redis: + host: redis-denorm.redis.svc.cluster.local + port: 6379 + cassandra: + host: localhost + port: 9042 + kafka: + host: "kafka-headless.kafka.svc.cluster.local" + port: 9092 + zookeeper: + host: "zookeeper-headless.kafka.svc.cluster.local" + port: 2181 + image: + registry: "" + +podAnnotations: {} + +podSecurityContext: + runAsNonRoot: true + runAsUser: 9999 + fsGroup: 0 + +securityContext: + {} + # readOnlyRootFilesystem: false + # capabilities: + # drop: + # - ALL + +service: + type: ClusterIP + ports: + - name: http + port: 8081 + targetPort: 8081 + +ingress: + enabled: false + annotations: {} + hosts: + - host: chart-example.local + paths: + - / + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +configmap: + enabled: false + mountPath: /config + +serviceAccount: + create: true + name: "" + +# Example values.yaml structure +initContainers: + {} + # - name: init-myservice + # image: busybox:1.28 + # command: ['sh', '-c', "until nslookup kubernetes.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] + +sidecars: + {} + # - name: log-reader # Sidecar container + # image: busybox # Use another busybox image + # command: ["/bin/sh"] # Override the default command + # args: ["-c", "tail -f /var/log/app.log"] # Run a shell script that tails the log file + +opa: + enabled: false + +jobmanager: + rpc_port: 6123 + blob_port: 6124 + query_port: 6125 + ui_port: 8081 + prom_port: 9250 + heap_memory: 1024 + +rest_port: 80 +resttcp_port: 8081 + +taskmanager: + prom_port: 9251 + rpc_port: 6122 + heap_memory: 1024 + replicas: 1 + cpu_requests: 0.3 + +checkpoint_store_type: null + +# AWS S3 Details +s3_auth_type: "serviceAccount" +s3_access_key: "" +s3_secret_key: "" +s3_endpoint: "" +s3_path_style_access: "" + +# Azure Container Details +azure_account: "azure-test" +azure_secret: "azure-secret" + +# Azure Container Details +cloud_storage_flink_bucketname: flink-state-backend +cloud_storage_content_bucketname: sunbird-content-dev +cert_container_name: dev-e-credentials +# cloud_storage_endpoint: https://{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net + +postgres: + max_connections: 2 + sslmode: false + db_name: analytics + db_port: 5432 + +checkpointing: + enabled: false + statebackend: null + +log4j_console_properties: | + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + logger.flink.name = org.apache.flink + logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = ERROR + logger.kafka.name= org.apache.kafka + logger.kafka.level = ERROR + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = ERROR + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = ERROR + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=10MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 5 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF + +flink_resources: + taskmanager: + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1 + memory: 1024Mi + jobmanager: + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1 + memory: 1024Mi + +serviceMonitor: + jobmanager: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: {} # additional labels e.g. release: prometheus + honorLabels: true + jobLabel: "app.kubernetes.io/name" + port: prom + taskmanager: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: {} # additional labels e.g. release: prometheus + honorLabels: true + jobLabel: "app.kubernetes.io/name" + port: prom + +flink_jobs: + "kafka-connector-1-0-0": + "enabled": "false" + "connector_id": "kafka-connector-1.0.0" + "source": "kafka-connector-1.0.0" + "main_program": "kafka-connector-1.0.0.jar" + +commonAnnotations: + reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/Chart.lock b/command-service/helm-charts/spark-connector-cron/Chart.lock new file mode 100644 index 00000000..13768315 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://nimbushubin.github.io/helmcharts + version: 0.1.0 +digest: sha256:1caa4d36f25ec305383122b0a8a6d585921a608b03d9aaf15fa70b5044d109ea +generated: "2023-10-10T06:58:15.158733765+02:00" diff --git a/command-service/helm-charts/spark-connector-cron/Chart.yaml b/command-service/helm-charts/spark-connector-cron/Chart.yaml new file mode 100644 index 00000000..7c047bec --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +appVersion: 1.0.0 +dependencies: +- name: common + repository: https://nimbushubin.github.io/helmcharts + version: 0.1.0 +description: A production-ready Helm chart base template +maintainers: +- name: NimbusHub.in +name: spark-connector-cron +version: 0.1.0 diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/.helmignore b/command-service/helm-charts/spark-connector-cron/charts/common/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml b/command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml new file mode 100644 index 00000000..cc1e9676 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +description: Library chart for Sunbird +keywords: +- common +- helper +- template +- function +maintainers: +- name: NimbusHub.in +name: common +type: library +version: 0.1.0 diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..e85b1df4 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl @@ -0,0 +1,139 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a topologyKey definition +{{ include "common.affinities.topologyKey" (dict "topologyKey" "BAR") -}} +*/}} +{{- define "common.affinities.topologyKey" -}} +{{ .topologyKey | default "kubernetes.io/hostname" -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: 1 + {{- range $extraPodAffinityTerms }} + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: {{ .weight | default 1 -}} + {{- end -}} +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- range $extraPodAffinityTerms }} + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- end -}} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..b1257397 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl @@ -0,0 +1,229 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for daemonset. +*/}} +{{- define "common.capabilities.daemonset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Vertical Pod Autoscaler. +*/}} +{{- define "common.capabilities.vpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if PodSecurityPolicy is supported +*/}} +{{- define "common.capabilities.psp.supported" -}} +{{- if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if AdmissionConfiguration is supported +*/}} +{{- define "common.capabilities.admisionConfiguration.supported" -}} +{{- if semverCompare ">=1.23-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for AdmissionConfiguration. +*/}} +{{- define "common.capabilities.admisionConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiserver.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodSecurityConfiguration. +*/}} +{{- define "common.capabilities.podSecurityConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "pod-security.admission.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_configs.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_configs.tpl new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..07ded6f6 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl new file mode 100644 index 00000000..1bcb779d --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl @@ -0,0 +1,117 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" .Values.global ) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $separator := ":" -}} +{{- $termination := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .imageRoot.digest }} + {{- $separator = "@" -}} + {{- $termination = .imageRoot.digest | toString -}} +{{- end -}} +{{- if $registryName }} + {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}} +{{- else -}} + {{- printf "%s%s%s" $repositoryName $separator $termination -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end }} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper image version (ingores image revision/prerelease info & fallbacks to chart appVersion) +{{ include "common.images.version" ( dict "imageRoot" .Values.path.to.the.image "chart" .Chart ) }} +*/}} +{{- define "common.images.version" -}} +{{- $imageTag := .imageRoot.tag | toString -}} +{{/* regexp from https://github.com/Masterminds/semver/blob/23f51de38a0866c5ef0bfc42b3f735c73107b700/version.go#L41-L44 */}} +{{- if regexMatch `^([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?$` $imageTag -}} + {{- $version := semver $imageTag -}} + {{- printf "%d.%d.%d" $version.Major $version.Minor $version.Patch -}} +{{- else -}} + {{- print .chart.AppVersion -}} +{{- end -}} +{{- end -}} + diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..efa5b85c --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl @@ -0,0 +1,73 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") (hasKey .annotations "kubernetes.io/tls-acme") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..d90a6cdc --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl @@ -0,0 +1,46 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Kubernetes standard labels +{{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "common.labels.standard" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{- $default := dict "app.kubernetes.io/name" (include "common.names.name" .context) "helm.sh/chart" (include "common.names.chart" .context) "app.kubernetes.io/instance" .context.Release.Name "app.kubernetes.io/managed-by" .context.Release.Service -}} +{{- with .context.Chart.AppVersion -}} +{{- $_ := set $default "app.kubernetes.io/version" . -}} +{{- end -}} +{{ template "common.tplvalues.merge" (dict "values" (list .customLabels $default) "context" .context) }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Labels used on immutable fields such as deploy.spec.selector.matchLabels or svc.spec.selector +{{ include "common.labels.matchLabels" (dict "customLabels" .Values.podLabels "context" $) -}} + +We don't want to loop over custom labels appending them to the selector +since it's very likely that it will break deployments, services, etc. +However, it's important to overwrite the standard labels if the user +overwrote them on metadata.labels fields. +*/}} +{{- define "common.labels.matchLabels" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{ merge (pick (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) "app.kubernetes.io/name" "app.kubernetes.io/instance") (dict "app.kubernetes.io/name" (include "common.names.name" .context) "app.kubernetes.io/instance" .context.Release.Name ) | toYaml }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl new file mode 100644 index 00000000..a222924f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl @@ -0,0 +1,71 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a193c46b --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl @@ -0,0 +1,172 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + - failOnNew - Boolean - Optional - Default to true. If set to false, skip errors adding new keys to existing secrets. +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $failOnNew := default true .failOnNew }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key | quote }} + {{- else if $failOnNew }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Reuses the value from an existing secret, otherwise sets its value to a default value. + +Usage: +{{ include "common.secrets.lookup" (dict "secret" "secret-name" "key" "keyName" "defaultValue" .Values.myValue "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - defaultValue - String - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - context - Context - Required - Parent context. + +*/}} +{{- define "common.secrets.lookup" -}} +{{- $value := "" -}} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data -}} +{{- if and $secretData (hasKey $secretData .key) -}} + {{- $value = index $secretData .key -}} +{{- else if .defaultValue -}} + {{- $value = .defaultValue | toString | b64enc -}} +{{- end -}} +{{- if $value -}} +{{- printf "%s" $value -}} +{{- end -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..16405a0f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..a8ed7637 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,38 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + +{{/* +Merge a list of values that contains template after rendering them. +Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge +Usage: +{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }} +*/}} +{{- define "common.tplvalues.merge" -}} +{{- $dst := dict -}} +{{- range .values -}} +{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}} +{{- end -}} +{{ $dst | toYaml }} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..bfbddf05 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ include "common.names.namespace" .context | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Checksum a template at "path" containing a *single* resource (ConfigMap,Secret) for use in pod annotations, excluding the metadata (see #18376). +Usage: +{{ include "common.utils.checksumTemplate" (dict "path" "/configmap.yaml" "context" $) }} +*/}} +{{- define "common.utils.checksumTemplate" -}} +{{- $obj := include (print .context.Template.BasePath .path) .context | fromYaml -}} +{{ omit $obj "apiVersion" "kind" "metadata" | toYaml | sha256sum }} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl new file mode 100644 index 00000000..e5572991 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl @@ -0,0 +1,41 @@ +{{/* Define findkey function */}} +{{- define "common.variables.findkey" -}} +{{- $value := .value -}} +{{- $keys := .keys -}} +{{- $found := true -}} + +{{- range $key := $keys -}} + {{- if kindIs "map" $value -}} + {{- if hasKey $value $key -}} + {{- $value = index $value $key -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} +{{- end -}} + +{{- if $found -}} + {{- $value -}} +{{- else -}} + {{- "" -}} +{{- end -}} +{{- end -}} + +{{/* Define getmethekey function with precedence and input as a dot-delimited string using findkey */}} +{{- define "common.variables.variableGlobal" -}} +{{- $root := first . -}} +{{- $keyString := index (rest .) 0 -}} +{{- $keys := splitList "." $keyString -}} + +{{- $value := include "common.variables.findkey" (dict "value" $root.Values "keys" $keys) -}} + +{{- if (not $value) -}} + {{- $value = include "common.variables.findkey" (dict "value" $root.Values.global "keys" $keys) -}} +{{- end -}} + +{{- $value -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..66dffc1f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl @@ -0,0 +1,19 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..eda9aada --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..17d83a2f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..bbb445b8 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,113 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..ca3953f8 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..8c9aa570 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,134 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..fc0d208d --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,81 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..31ceda87 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,51 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/values.yaml b/command-service/helm-charts/spark-connector-cron/charts/common/values.yaml new file mode 100644 index 00000000..c83b33f2 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/values.yaml @@ -0,0 +1,82 @@ +# Default values for common. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/command-service/helm-charts/spark-connector-cron/templates/NOTES.txt b/command-service/helm-charts/spark-connector-cron/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl b/command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl new file mode 100644 index 00000000..14cd7f2d --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl @@ -0,0 +1,4 @@ +{{- define "base.serviceaccountname" -}} + {{- $name := printf "%s-%s" .Chart.Name "sa" }} + {{- default $name .Values.serviceAccount.name }} +{{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl new file mode 100644 index 00000000..f4b82f3c --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl @@ -0,0 +1,4 @@ +{{- define "base.cronReleaseName" -}} + {{- $name := printf "%s--%s" .Chart.Name .Values.instance_id }} + {{- default .Values.instance_id $name }} +{{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/_image.tpl b/command-service/helm-charts/spark-connector-cron/templates/_image.tpl new file mode 100644 index 00000000..0601533c --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_image.tpl @@ -0,0 +1,10 @@ +{{- define "base.image" }} +{{- $registry := default .Values.global.image.registry .Values.registry }} +{{- $image := printf "%s/%s" $registry .Values.repository}} +{{- if .Values.digest }} +{{- printf "%s@%s" $image .Values.digest }} +{{- else }} +{{- $tag := default "latest" .Values.tag }} +{{- printf "%s:%s" $image $tag }} +{{- end }} +{{- end }} diff --git a/command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl b/command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl new file mode 100644 index 00000000..4016b1e4 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl @@ -0,0 +1,14 @@ +# .Values.namespace will get overridden by .Values.global.namespace.chart-name +{{- define "base.namespace" -}} + {{- $chartName := .Chart.Name }} + {{- $namespace := default .Release.Namespace .Values.namespace }} + {{- if .Values.global }} + {{- with .Values.global.namespace }} + {{- if hasKey . $chartName }} + {{- $namespace = index . $chartName }} + {{- end }} + {{- end }} + {{- end }} + {{- $namespace | trunc 63 | trimSuffix "-" }} +{{- end }} + diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml new file mode 100644 index 00000000..7d35b32b --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -0,0 +1,83 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "base.cronReleaseName" . }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + concurrencyPolicy: Forbid + schedule: {{ .Values.cronSchedule }} + jobTemplate: + spec: + template: + metadata: + annotations: + {{- if .Values.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 12 }} + {{- end }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name }} + restartPolicy: {{ .Values.restartPolicy }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{- include "base.image" .}}" + imagePullPolicy: {{ .Values.imagePullPolicy }} + {{- if .Values.livenessProbe }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 16 }} + {{- end }} + {{- if .Values.readinessProbe }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 16 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 16 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 16 }} + {{- if .Values.configmap.enabled }} + envFrom: + - configMapRef: + name: {{ include "common.names.fullname" . }}-env + volumeMounts: + - name: config + mountPath: {{ .Values.configmap.mountPath }} + {{- end }} + command: + {{- if eq .Values.technology "scala" }} + - /bin/sh + - -c + - |+ + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\*.jar --class {{ .Values.main_class }} /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-scala-config.conf -c {{ .Values.instance_id }}" + {{- else if eq .Values.technology "python" }} + - /bin/sh + - -c + - | + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.configmap.enabled }} + volumes: + - name: config + configMap: + name: {{ include "common.names.fullname" . }} + {{- end }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml new file mode 100644 index 00000000..ceca090a --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -0,0 +1,153 @@ +nameOverride: "" +fullnameOverride: "" +namespace: "spark" + +replicaCount: 1 + +global: + image: + registry: "sanketikahub" + +registry: "bitnami" +repository: "kubectl" +tag: "latest" +digest: "" + +imagePullPolicy: IfNotPresent +imagePullSecrets: [] + +commonLabels: {} + +commonAnnotations: {} + +podAnnotations: {} + +podSecurityContext: {} + # runAsNonRoot: true + # runAsUser: 1001 + # fsGroup: 1001 + +securityContext: {} + # readOnlyRootFilesystem: false + # capabilities: + # drop: + # - ALL + +# This block is an interface for k8s service spec. +service: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: 80 + +ingress: + enabled: false + annotations: {} + hosts: + - paths: + - / + # host: chart-example.local + +resources: {} + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + +livenessProbe: {} + # httpGet: + # path: "/healthz" + # port: 8080 + # initialDelaySeconds: 5 + # periodSeconds: 5 + +readinessProbe: {} + # httpGet: + # path: "/ready" + # port: 8080 + # initialDelaySeconds: 5 + # periodSeconds: 5 + +nodeSelector: {} +tolerations: [] +affinity: {} + +configmap: + enabled: false + mountPath: /config + +serviceAccount: + name: spark-connectors-sa + +serviceMonitor: + enabled: false + selectorLabels: + release: monitoring + endpoints: [] + # - port: http # the name of the port in your service, assuming the primary service port is named 'http' in this example. + # path: /metrics + # interval: 30s + # scrapeTimeout: 10s + # honorLabels: true + +# Example values.yaml structure +initContainers: {} + # - name: init-myservice + # image: busybox:1.28 + # command: ['sh', '-c', "until nslookup kubernetes.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] + +sidecars: {} + # - name: log-reader # Sidecar container + # image: busybox # Use another busybox image + # command: ["/bin/sh"] # Override the default command + # args: ["-c", "tail -f /var/log/app.log"] # Run a shell script that tails the log file + +### NOTE: This section isn't to be changed and must be in sync with the spark helm chart +######## Spark Configuration Start ######## +## Defaults taken from Bitnami Spark Helm Chart +python_path: /opt/bitnami/python/bin/python + +spark: + host: http://spark-master-svc.spark.svc.cluster.local:8998/batches/ + master: + host: spark://spark-master-0.spark-headless.spark.svc.cluster.local:7077 + port: 7077 + driver: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties + executor: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties +######## Spark Configuration End ######## + + + +## JDBC Connector +technology: scala +connector_source: jdbc-connector-1.0.0 +instance_id: nyt-psql.1 +main_class: org.sunbird.obsrv.connector.JDBCConnector +main_file: jdbc-connector-1.0.0.jar + +## Object Store Connector +# technology: python +# connector-source: object_store_connector-0.1.0 +# instance-id: s3.new-york-taxi-data.1 +# main_file: object_store_connector/__main__.py + +# The schedule in cron format +cronSchedule: 0 0 0 * * # Every month +restartPolicy: OnFailure \ No newline at end of file diff --git a/command-service/requirements.txt b/command-service/requirements.txt index 8781cb5f..6136ac6d 100644 --- a/command-service/requirements.txt +++ b/command-service/requirements.txt @@ -1,6 +1,19 @@ fastapi==0.103.0 uvicorn==0.20.0 dataclasses-json==0.5.7 -urllib3==2.0.7 +# pydantic==1.10.5 pyyaml==6.0.1 +urllib3==2.0.7 +# protobuf==3.20.* +# pyhelm==2.14.5 +# kubernetes==28.1.0 +# helm_operator_sdk +# psycopg2==2.9.5 +psycopg2-binary +dacite==1.8.0 backoff==2.2.1 +tenacity==8.2.2 +kafka-python-ng==2.2.2 +boto3 +prometheus-client +requests \ No newline at end of file diff --git a/command-service/src/command/__init__.py b/command-service/src/command/__init__.py index 56523041..7b89c048 100644 --- a/command-service/src/command/__init__.py +++ b/command-service/src/command/__init__.py @@ -1,3 +1,7 @@ +# autoflake: skip_file from .command_executor import CommandExecutor +from .connector_registry import ConnectorRegistry +from .db_command import DBCommand +from .druid_command import DruidCommand from .flink_command import FlinkCommand -from .icommand import ICommand \ No newline at end of file +from .icommand import ICommand diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py new file mode 100644 index 00000000..1c989a5c --- /dev/null +++ b/command-service/src/command/alert_manager_command.py @@ -0,0 +1,215 @@ +import json + +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.db_service import DatabaseService +from service.http_service import HttpService + + +class AlertManagerService(ICommand): + + def __init__( + self, config: Config, db_service: DatabaseService, http_service: HttpService + ): + self.config = config + self.db_service = db_service + self.http_service = http_service + self.metrics = self.config.find("alert_manager.metrics") + self.masterdata_metrics = self.config.find("alert_manager.masterdata_metrics") + self.object_connector_metrics = self.config.find( + "alert_manager.object_connector_metrics" + ) + self.jdbc_connector_metrics = self.config.find( + "alert_manager.jdbc_connector_metrics" + ) + self.config_service_host = self.config.find("config_service.host") + self.config_service_port = self.config.find("config_service.port") + self.base_url = ( + f"http://{self.config_service_host}:{self.config_service_port}/alerts/v1" + ) + + def execute(self, command_payload: CommandPayload, action: Action): + dataset = self.get_dataset(dataset_id=command_payload.dataset_id) + if dataset is None: + return ActionResponse( + status="ERROR", + status_code=500, + error_message=f"Dataset {command_payload.dataset_id} does not exist", + ) + + dataset_source_config = self.get_dataset_source_config( + dataset_id=command_payload.dataset_id + ) + + for metric in self.metrics: + for service, metrics in metric.items(): + for metric_info in metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric_info, + dataset_name=dataset["name"], + ) + + for config in dataset_source_config: + if config["connector_type"] == "object": + for metric in self.object_connector_metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric, + dataset_name=dataset["name"], + ) + if config["connector_type"] == "jdbc": + for metric in self.jdbc_connector_metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric, + dataset_name=dataset["name"], + ) + + if dataset["type"] == "master-dataset": + for metric_info in self.masterdata_metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric_info, + dataset_name=dataset["name"], + ) + return ActionResponse(status="OK", status_code=200) + + def get_dataset(self, dataset_id: str) -> str: + query = f"SELECT * FROM datasets WHERE dataset_id= %s" + params = (dataset_id,) + result = self.db_service.execute_select_one(sql=query, params=params) + return result + + def get_dataset_source_config(self, dataset_id: str) -> str: + query = f"SELECT * FROM dataset_source_config WHERE dataset_id= %s" + params = (dataset_id,) + result = self.db_service.execute_select_all(sql=query, params=params) + return result + + def get_modified_metric( + self, service: str, metric: dict, payload: CommandPayload + ) -> dict: + if service == "flink": + substring = f"{payload.dataset_id}" + modified_substring = substring.replace("-", "_") + modified_metric = metric["metric"].replace("dataset_id", modified_substring) + metric["metric"] = modified_metric + return metric + else: + metric["metric"] = metric["metric"].replace( + "dataset_id", payload.dataset_id + ) + return metric + + def create_alert_metric( + self, payload: CommandPayload, service: str, metric: dict, dataset_name: str + ) -> ActionResponse: + metric_url = f"{self.base_url}/metric/alias/create" + + metric_data = self.get_modified_metric( + service=service, metric=metric, payload=payload + ) + + prom_metric = metric_data["metric"] + metric_alias = f"{metric_data['alias']} ({payload.dataset_id})" + # Metric api payload + metric_body = json.dumps( + { + "alias": metric_alias, + "component": "datasets", + "subComponent": dataset_name, + "metric": prom_metric, + "context": { + "datasetId": payload.dataset_id, + }, + } + ) + + response = self.http_service.post( + url=metric_url, + body=metric_body, + headers={"Content-Type": "application/json"}, + ) + if response.status == 200: + self.create_alert_rule( + payload={"dataset_name": dataset_name, "metric_data": metric_data} + ) + else: + error_data = json.loads(response.body) + error_message = error_data["params"]["errmsg"] + return ActionResponse( + status="ERROR", + status_code=500, + error_message=f"Error creating alert metric {metric_alias}: {error_message}", + ) + + def create_alert_rule(self, payload: dict) -> ActionResponse: + dataset_name = payload["dataset_name"] + prom_metric = payload["metric_data"]["metric"] + description = payload["metric_data"]["description"] + metric_alias = payload["metric_data"]["alias"] + frequency = payload["metric_data"]["frequency"] + interval = payload["metric_data"]["interval"] + operator = payload["metric_data"]["operator"] + threshold = payload["metric_data"]["threshold"] + + alert_body = json.dumps( + { + "name": f"{dataset_name}_{metric_alias}", + "manager": "grafana", + "description": description + or f"Automated alert set up for dataset {dataset_name}", + "category": "datasets", + "frequency": frequency, + "interval": interval, + "context": {"alertType": "SYSTEM"}, + "labels": {"component": "obsrv"}, + "metadata": { + "queryBuilderContext": { + "category": "datasets", + "subComponent": dataset_name, + "metric": prom_metric, + "operator": operator, + "threshold": threshold, + "metricAlias": metric_alias, + } + }, + "notification": {"channels": []}, + } + ) + + response = self.http_service.post( + url=f"{self.base_url}/create", + body=alert_body, + headers={"Content-Type": "application/json"}, + ) + result = json.loads(response.body) + alert_id = result["result"]["id"] + if response.status == 200: + self.publish_alert_rule(alert_id=alert_id) + else: + error_data = json.loads(response.body) + error_message = error_data["params"]["errmsg"] + return ActionResponse( + status="ERROR", + status_code=500, + error_message=f"Error creating alert rule for {dataset_name}: {error_message}", + ) + + def publish_alert_rule(self, alert_id: str) -> ActionResponse: + endpoint = f"/publish/{alert_id}" + url = self.base_url + endpoint + try: + response = self.http_service.get(url=url) + except Exception as e: + return ActionResponse( + status="ERROR", + status_code=500, + message=f"Error publishing alert rule for {alert_id}: {e}", + ) diff --git a/command-service/src/command/command_executor.py b/command-service/src/command/command_executor.py index ea0188d6..665e4636 100644 --- a/command-service/src/command/command_executor.py +++ b/command-service/src/command/command_executor.py @@ -1,34 +1,108 @@ +import logging + +import psycopg2 +from urllib3.exceptions import MaxRetryError, NewConnectionError + +from command.alert_manager_command import AlertManagerService +from command.connector_command import ConnectorCommand +from command.dataset_command import DatasetCommand +from command.db_command import DBCommand +from command.druid_command import DruidCommand from command.flink_command import FlinkCommand -from exception.exception import FlinkHelmInstallException, FlinkConnectionException, HttpConnectionException -from service.http_service import HttpService +from command.telemetry_command import TelemetryCommand from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.db_service import DatabaseService +from service.http_service import HttpService +from service.telemetry_service import TelemetryService -from model.data_models import Action, CommandPayload, ActionResponse -import logging -class CommandExecutor(): +class CommandExecutor: def __init__(self): self.config_obj = Config() + self.db_service = DatabaseService() self.http_service = HttpService() - self.flink_command = FlinkCommand(config=self.config_obj, http_service=self.http_service) + self.telemetry_service = TelemetryService() + self.flink_command = FlinkCommand( + config=self.config_obj, http_service=self.http_service + ) + self.connector_command = ConnectorCommand( + config=self.config_obj, db_service=self.db_service + ) + self.druid_command = DruidCommand( + config=self.config_obj, + db_service=self.db_service, + http_service=self.http_service, + ) + self.alert_manager_command = AlertManagerService( + config=self.config_obj, + db_service=self.db_service, + http_service=self.http_service, + ) + self.dataset_command = DatasetCommand( + db_service=self.db_service, + telemetry_service=self.telemetry_service, + http_service=self.http_service, + config=self.config_obj, + ) + self.db_command = DBCommand( + db_service=self.db_service, dataset_command=self.dataset_command + ) + self.audit_event_command = TelemetryCommand( + telemetry_service=self.telemetry_service, + dataset_command=self.dataset_command, + ) self.action_commands = {} - self.action_commands[Action.RESTART_PIPELINE_JOBS.name] = self.flink_command + self.action_commands[Action.START_PIPELINE_JOBS.name] = self.flink_command + self.action_commands[Action.MAKE_DATASET_LIVE.name] = self.db_command + self.action_commands[Action.SUBMIT_INGESTION_TASKS.name] = self.druid_command + self.action_commands[Action.DEPLOY_CONNECTORS.name] = self.connector_command + self.action_commands[Action.CREATE_ALERT_METRIC.name] = ( + self.alert_manager_command + ) + self.action_commands[Action.CREATE_AUDIT_EVENT.name] = self.audit_event_command self.logger = logging.getLogger() - - def execute_command(self, payload: CommandPayload): + + def execute_command(self, payload: CommandPayload, ts: int): command = payload.command.name workflow_commands = self.get_command_workflow(command) + print(workflow_commands) for sub_command in workflow_commands: command = self.action_commands[sub_command] print(f"Executing command {sub_command}") try: - result = command.execute(command_payload=payload, action=sub_command) - except (FlinkHelmInstallException, FlinkConnectionException, HttpConnectionException) as conn_error: - self.logger.exception("Error when trying to connect to flink service...", conn_error) - result = ActionResponse(status="ERROR", status_code=500, error_message="HTTP_CONNECTION_ERROR") + if sub_command == Action.CREATE_AUDIT_EVENT.name: + command.execute(command_payload=payload, action=sub_command, ts=ts) + else: + result = command.execute( + command_payload=payload, action=sub_command + ) + except ( + ConnectionRefusedError, + MaxRetryError, + NewConnectionError, + ) as conn_error: + self.logger.exception( + "Error when trying to connect to http endpoint...", conn_error + ) + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="HTTP_CONNECTION_ERROR", + ) + return result + except psycopg2.OperationalError as db_conn_error: + self.logger.exception( + "Error when trying to connect to database...", db_conn_error + ) + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="DATABASE_CONNECTION_ERROR", + ) return result - return result + return result def get_command_workflow(self, action: Action): - return self.config_obj.find("commands.{0}.workflow".format(action)) \ No newline at end of file + return self.config_obj.find("commands.{0}.workflow".format(action)) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py new file mode 100644 index 00000000..aabb9db4 --- /dev/null +++ b/command-service/src/command/connector_command.py @@ -0,0 +1,370 @@ +from dacite import from_dict +import json +import logging +import subprocess + +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from model.db_models import ConnectorInstance +from service.db_service import DatabaseService + + +class ConnectorCommand(ICommand): + def __init__(self, config: Config, db_service: DatabaseService): + self.config = config + self.db_service = db_service + self.logger = logging.getLogger() + self.connector_job_config = self.config.find("connector_jobs") + + def execute(self, command_payload: CommandPayload, action: Action): + result = None + active_connectors = self._get_connector_details( + dataset_id=command_payload.dataset_id + ) + is_masterdata = self._get_masterdata_details( + dataset_id=command_payload.dataset_id + ) + + print(f"Active connectors: {active_connectors}") + print(f"Is masterdata: {is_masterdata}") + + if action == Action.DEPLOY_CONNECTORS.name: + result = self._deploy_connectors( + command_payload.dataset_id, active_connectors, is_masterdata + ) + + return result + + def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): + result = None + self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) + result = self._install_jobs(dataset_id, active_connectors, is_masterdata) + + return result + + def _stop_connector_jobs(self, is_masterdata, namespace): + print(f"Uninstalling jobs for {namespace}..") + base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] + + # managed_releases = [] + # connector_jar_config = self.config.find("connector_job") + # masterdata_jar_config = self.config.find("masterdata_job") + # for connector_type in connector_jar_config: + # for release in connector_jar_config[connector_type]: + # managed_releases.append(release["release_name"]) + # if is_masterdata: + # for release in masterdata_jar_config: + # managed_releases.append(release["release_name"]) + + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] + helm_ls_result = subprocess.run( + helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + for job in jobs.splitlines()[1:]: + release_name = job.split()[0] + if base_helm_chart in job: + print("Uninstalling job {0}".format(release_name)) + helm_uninstall_cmd = [ + "helm", + "uninstall", + release_name, + "--namespace", + namespace, + ] + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled job {release_name}...") + else: + print( + f"Error uninstalling job {release_name}: {helm_uninstall_result.stderr.decode()}" + ) + + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): + result = None + for connector in active_connectors: + print("Installing connector {0}".format(connector)) + + if connector.connector_runtime == "spark": + result = self._perform_spark_install(dataset_id, connector) + elif connector.connector_runtime == "flink": + result = self._perform_flink_install(dataset_id, connector) + else: + print( + f"Connector {connector.connector_id} is not supported for deployment" + ) + break + + # if is_masterdata: + # print("Installing masterdata job") + # masterdata_jar_config = self.config.find("masterdata_job") + # for release in masterdata_jar_config: + # result = self._perform_install(release) + return result + + def _perform_flink_install(self, dataset_id, connector_instance): + err = None + result = None + release_name = connector_instance.connector_id + runtime = connector_instance.connector_runtime + namespace = self.connector_job_config["flink"]["namespace"] + job_name = release_name.replace(".", "-") + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] + + helm_ls_result = subprocess.run( + helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) + if deployment_exists: + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, job_name=job_name + ) + print("Restart command: ", restart_cmd) + # Run the helm command + helm_install_result = subprocess.run( + restart_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + if helm_install_result.returncode == 0: + print(f"Job {job_name} restart succeeded...") + else: + err = True + return ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_LIST_EXCEPTION", + ) + print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result + else: + if self._get_live_instances(runtime="flink", connector_instance=connector_instance): + connector_source = connector_instance.connector_source + flink_jobs = dict() + flink_jobs[job_name] = { + "enabled": "true", + "connector_id": connector_instance.connector_id, + "source": connector_source.get("source"), + "main_program": connector_source.get("main_program") + } + + set_json_value = json.dumps(flink_jobs) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + job_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set-json", + f"""flink_jobs={set_json_value.replace(" ", "")}""" + ] + + print("flink connector installation: ", " ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + print(helm_install_result) + + if helm_install_result.returncode == 0: + print(f"Job '{job_name}' deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error installing job '{job_name}': {helm_install_result.stderr.decode()}" + ) + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result + else: + self._stop_connector_jobs(is_masterdata=False, namespace="flink") + else: + print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") + return ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_LIST_EXCEPTION", + ) + + def _perform_spark_install(self, dataset_id, connector_instance): + err = None + result = None + release_name = connector_instance.id + connector_source = connector_instance.connector_source + schedule = connector_instance.operations_config["schedule"] + + schedule_configs = { + "Hourly": "0 * * * *", # Runs at the start of every hour + "Weekly": "0 0 * * 0", # Runs at midnight every Sunday + "Monthly": "0 0 1 * *", # Runs at midnight on the 1st day of every month + "Yearly": "0 0 1 1 *" # Runs at midnight on January 1st each year + } + + namespace = self.connector_job_config["spark"]["namespace"] + + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print("spark connector installation:", " ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job '{release_name}' update succeeded...") + result = ActionResponse(status="OK", status_code=200) + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", + ) + print(f"Error updating job '{release_name}': {helm_install_result.stderr.decode()}") + + if result is None: + result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") + + return result + + def _get_connector_details(self, dataset_id): + active_connectors = [] + query = f""" + SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version + FROM connector_instances ci + JOIN connector_registry cr on ci.connector_id = cr.id + WHERE ci.status= %s and ci.dataset_id = %s + """ + params = (DatasetStatusType.Live.name, dataset_id,) + records = self.db_service.execute_select_all(sql=query, params=params) + + for record in records: + active_connectors.append(from_dict( + data_class=ConnectorInstance, data=record + )) + + return active_connectors + + def _get_masterdata_details(self, dataset_id): + is_masterdata = False + query = f""" + SELECT * + FROM datasets + WHERE status= %s AND dataset_id = %s AND type = 'master' + """ + params = (DatasetStatusType.Live.name, dataset_id,) + rows = self.db_service.execute_select_all(sql=query, params=params) + if len(rows) > 0: + is_masterdata = True + + return is_masterdata + + ## TODO: check for connector_id as well + def _get_live_instances(self, runtime, connector_instance): + has_live_instances = False + query = f""" + SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id + FROM connector_instances ci + JOIN connector_registry cr ON ci.connector_id = cr.id + JOIN datasets d ON ci.dataset_id = d.id + WHERE cr.runtime = %s AND ci.status = %s AND ci.connector_id = %s; + """ + params = (runtime, DatasetStatusType.Live.name, connector_instance.connector_id) + rows = self.db_service.execute_select_all(sql=query, params=params) + if len(rows) > 0: + has_live_instances = True + + return has_live_instances + + # def _perform_install(self, release): + # err = None + # result = None + # release_name = release["release_name"] + # helm_install_cmd = [ + # "helm", + # "upgrade", + # "--install", + # release_name, + # self.connector_job_chart_dir, + # "--namespace", + # self.connector_job_ns, + # "--create-namespace", + # "--set", + # "file.path={}".format(release["jar"]), + # "--set", + # "class.name={}".format(release["class"]), + # "--set", + # "job.name={}".format(release_name), + # "--set", + # "args={}".format(",".join(release["args"])), + # "--set", + # "schedule={}".format(release["schedule"]), + # ] + # helm_install_result = subprocess.run( + # helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + # ) + # if helm_install_result.returncode == 0: + # print(f"Job {release_name} deployment succeeded...") + # else: + # err = True + # result = ActionResponse( + # status="ERROR", + # status_code=500, + # error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + # ) + # print( + # f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + # ) + + # if err is None: + # result = ActionResponse(status="OK", status_code=200) + + # return result + diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py new file mode 100644 index 00000000..ceb2cbba --- /dev/null +++ b/command-service/src/command/connector_registry.py @@ -0,0 +1,563 @@ +import json +import os +import base64 +import tarfile +import uuid +import zipfile +from datetime import datetime +from pathlib import Path + +import requests +import subprocess +from fastapi import status + +from config import Config +from service.http_service import HttpService +from service.db_service import DatabaseService +from model.db_models import ConnectorRegsitryv2 + + +class RegistryResponse: + def __init__(self, status: str, message: str, statusCode: status, connector_info = None): + self.status = status + self.connector_info = connector_info + self.message = message # Ensure message is directly a string + self.statusCode = statusCode + + +class ConnectorRegistry: + def __init__(self): + self.metadata = dict() + self.db_service = DatabaseService() + self.config = Config() + self.download_path = self.config.find("connector_registry.download_path") + self.uuid = str(uuid.uuid4()) + self.extraction_path = os.path.join(self.download_path, self.uuid) + self.metadata = None + self.ui_spec = None + self.metadata_file_name = self.config.find( + "connector_registry.metadata_file_name" + ) + self.ui_spec_file_name = self.config.find("connector_registry.ui_spec_file") + self.dataset_api_url = self.config.find("dataset_api.host").strip("/") + self.pre_signed_url = self.config.find("dataset_api.pre_signed_url").strip("/") + + def register(self, rel_path: str) -> RegistryResponse: + try: + download_file_path = os.path.join(self.download_path, rel_path) + file_extension = rel_path.split(".")[-1] + + # if not os.path.exists(download_file_path): + http_service = HttpService() + print( + f"Connector Registry | Received request to register connector: {rel_path} | {self.dataset_api_url}/{self.pre_signed_url}" + ) + dataset_api_request = {"request": {"files": [rel_path], "access": "read", "type": "connector"}} + dataset_api_response = http_service.post( + url=f"{self.dataset_api_url}/{self.pre_signed_url}", + body=json.dumps(dataset_api_request), + headers={"Content-Type": "application/json"} + ) + print( + f"Connector Registry | Dataset API Response: {dataset_api_response.body}" + ) + + if dataset_api_response.status != 200: + return RegistryResponse( + status="failure", + message="dataset api failed to generate read url.", + statusCode=status.HTTP_400_BAD_REQUEST, + ) + + dataset_api_response_data = json.loads(dataset_api_response.body) + url = dataset_api_response_data.get("result", [{}])[0].get( + "preSignedUrl", None + ) + if not url: + return RegistryResponse( + status="failure", + message="dataset api failed to generate read url.", + statusCode=status.HTTP_400_BAD_REQUEST, + ) + + self.cleanup_download_path() + os.makedirs(self.download_path, exist_ok=True) + + # download the file + download_status = self.download_file(url, download_file_path) + print(f"Connector Registry | Download status: {download_status}") + if not download_status: + return RegistryResponse( + status="failure", + message="failed to download the file", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + # extract the file + os.makedirs(self.extraction_path, exist_ok=True) + extraction_status = ExtractionUtil.extract( + file=download_file_path, + extract_out_path=self.extraction_path, + ext=file_extension, + ) + if extraction_status.status.lower() == "failure": + return extraction_status + + load_metadata_status = self.load_metadata(self.extraction_path) + + # Loading of metadata + if not load_metadata_status: + return RegistryResponse( + status="failure", + message="unable to locate the metadata file", + statusCode=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + + # check if folder exists + connector_source = f"{self.metadata['metadata']['id']}-{self.metadata['metadata']['version']}" + if not os.path.exists(os.path.join(self.extraction_path, connector_source)): + return RegistryResponse( + status="failure", + message=f"connector source folder not found; expecting {connector_source} folder inside the archive", + statusCode=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + + load_ui_config_status = self.load_ui_metadata(self.extraction_path) + + # Loading of ui configurations + if not load_ui_config_status: + return RegistryResponse( + status="failure", + message="Unable to locate the ui configuration File", + statusCode=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + + # Process of the metadata + return self.process_metadata(rel_path, connector_source) + + except Exception as e: + print(f"Connector Registry | An error occurred: {e}") + return RegistryResponse( + status="failure", + message="Failed to register connector", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + # Method to clean up the local directory + def cleanup_download_path(self): + if os.path.exists(self.download_path): + os.system(f"rm -rf {self.download_path}/*") + + # Method to load the file data into object (ex: ui config file and metadata file) + def load_json_file(self, extract_out_path, file_name, attribute_name): + for root, dirs, files in os.walk(extract_out_path): + for name in files: + if name == file_name: + print(f"Connector Registry | {file_name} found") + with open(os.path.join(root, name)) as f: + data = json.load(f) + if not data: + print(f"Connector Registry | Empty {file_name} file") + return False + setattr(self, attribute_name, data) + return True + print(f"Connector Registry | {file_name} not found") + return False + + def load_metadata(self, extract_out_path): + return self.load_json_file( + extract_out_path, self.metadata_file_name, "metadata" + ) + + def load_ui_metadata(self, extract_out_path): + return self.load_json_file(extract_out_path, self.ui_spec_file_name, "ui_spec") + + # Method to save the metadata and ui_spec configuration into db + def process_metadata(self, rel_path, connector_source) -> RegistryResponse: + result = [] + tenant = self.metadata.get("metadata", {}).get("tenant", "") + + self.copy_connector_to_runtime(self.metadata['metadata']['runtime'], connector_source) + + if tenant == "multiple": + connector_objects = self.metadata["connectors"] + for obj in connector_objects: + source = { + "source": connector_source, + "main_class": obj["main_class"] if "main_class" in obj else self.metadata["metadata"]["main_class"], + "main_program": obj["main_program"] if "main_program" in obj else self.metadata["metadata"]["main_program"], + } + + connector_id = obj["id"].replace(" ", "-") + self.update_connector_registry(connector_id, self.metadata['metadata']['version']) + registry_meta = ConnectorRegsitryv2(connector_id, + obj['name'], + 'source', + self.metadata['metadata']['category'], + self.metadata['metadata']['version'], + obj['description'], + self.metadata['metadata']['technology'], + self.metadata['metadata']['runtime'], + self.metadata['metadata']['licence'], + self.metadata['metadata']['owner'], + self.load_file_bytes(obj["icon"]), + 'Live', + rel_path, + json.dumps(source), + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + self.ui_spec[obj["id"]] if obj["id"] in self.ui_spec else {}, + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ) + query, params = self.build_insert_query(registry_meta) + success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + + if not success: + return RegistryResponse( + status="failure", + message=f"Failed to register connector {connector_id}", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + result.append(registry_meta.to_dict()) + return RegistryResponse( + status="success", + connector_info=result, + message="Connectors registered successfully", + statusCode=status.HTTP_200_OK + ) + + else: + connector_id = ( + self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") + ) + self.update_connector_registry(connector_id, self.metadata['metadata']['version']) + source = { + "source": connector_source, + "main_class": self.metadata["metadata"]["main_class"], + "main_program": self.metadata["metadata"]["main_program"], + } + + registry_meta = ConnectorRegsitryv2( + connector_id, + self.metadata['metadata']['name'], + 'source', + self.metadata['metadata']['category'], + self.metadata['metadata']['version'], + self.metadata['metadata']['description'], + self.metadata['metadata']['technology'], + self.metadata['metadata']['runtime'], + self.metadata['metadata']['licence'], + self.metadata['metadata']['owner'], + self.load_file_bytes(self.metadata['metadata']["icon"]), + 'Live', + rel_path, + json.dumps(source), + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + self.ui_spec, + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ) + query, params = self.build_insert_query(registry_meta) + success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + + if not success: + return RegistryResponse( + status="failure", + message=f"Failed to register connector {connector_id}", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + return RegistryResponse( + status="success", + message="Connectors registered successfully", + connector_info=[registry_meta.to_dict()], + statusCode=status.HTTP_200_OK, + ) + + + def execute_query(self, query, params) -> bool: + try: + result = self.db_service.execute_upsert(sql=query, params=params) + return result > 0 # Assuming the result is the number of affected rows + except Exception as e: + print( + f"Connector Registry | An error occurred during the execution of Query: {e}" + ) + return False + + # Method to download the file from blob store + def download_file(self, url, destination) -> bool: + try: + print(f"destination {destination}") + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(destination, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + file.write(chunk) + + print( + f"Connector Registry | Download completed successfully. URL:{url} Destination: {destination}" + ) + return True + except requests.exceptions.HTTPError as http_err: + print( + f"Connector Registry | HTTP error occurred during the file download: {http_err}" + ) + return False + except Exception as e: + print( + f"Connector Registry | An unexpected error occurred during the file download: {e}" + ) + return False + + def build_insert_query(self, registry_meta: ConnectorRegsitryv2): + ui_spec_json = json.dumps(registry_meta.ui_spec) + query =f""" + INSERT INTO connector_registry ( + id, connector_id, name, type, category, version, description, + technology, runtime, licence, owner, iconurl, status, source_url, + source, ui_spec, created_by, updated_by, created_date, updated_date, live_date + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s + ) ON CONFLICT ( + connector_id, version + ) DO UPDATE SET + id = %s, + name = %s, + type = %s, + category = %s, + version = %s, + description = %s, + technology = %s, + runtime = %s, + licence = %s, + owner = %s, + iconurl = %s, + status = %s, + source_url = %s, + source = %s, + ui_spec = %s::jsonb, + updated_date = %s + ;; + """ + params = ( + registry_meta.id + "-" + registry_meta.version, + registry_meta.id, + registry_meta.name, + registry_meta.type, + registry_meta.category, + registry_meta.version, + registry_meta.description, + + registry_meta.technology, + registry_meta.runtime, + registry_meta.licence, + registry_meta.owner, + registry_meta.iconurl, + registry_meta.status, + registry_meta.source_url, + + registry_meta.source, + ui_spec_json, + 'SYSTEM', + 'SYSTEM', + datetime.now(), + datetime.now(), + datetime.now(), + + registry_meta.id + "-" + registry_meta.version, + registry_meta.name, + registry_meta.type, + registry_meta.category, + registry_meta.version, + registry_meta.description, + registry_meta.technology, + registry_meta.runtime, + registry_meta.licence, + registry_meta.owner, + registry_meta.iconurl, + registry_meta.status, + registry_meta.source_url, + registry_meta.source, + ui_spec_json, + datetime.now(), + ) + return query, params + + def load_file_bytes(self, rel_path: str) -> bytes | None: + file_path = Path(self.extraction_path) + for item in file_path.glob("*/{}".format(rel_path)): + try: + with open(item, 'rb') as file: + file_content = file.read() + encoded = base64.b64encode(file_content).decode("ascii") + except IsADirectoryError: + print( + f"Connector Registry | No value for icon URL given at metadata: {rel_path}" + ) + return None + except FileNotFoundError: + print( + f"Connector Registry | No file present at indicated relative path: {rel_path}" + ) + return None + except (ValueError or TypeError) as e: + print( + f"Connector Registry | File content not byte like: {e}" + ) + return None + return encoded + + def update_connector_registry(self, _id, ver): + try: + result = self.db_service.execute_upsert( + f"UPDATE connector_registry SET status = 'Retired', updated_date = now() WHERE connector_id = %s AND status = 'Live' AND version != %s", (_id, ver) + ) + print( + f"Connector Registry | Updated {result} existing rows with connector_id: {_id} and version: {ver}" + ) + except Exception as e: + print( + f"Connector Registry | An error occurred during the execution of Query: {e}" + ) + + def copy_connector_to_runtime(self, runtime: str, connector_source: str): + if runtime == "spark": + return self.copy_connector_to_spark(connector_source) + + + def copy_connector_to_spark(self, connector_source: str): + print(f"Connector Registry | copying {connector_source} to spark") + ## get name of the spark pod using kubectl + spark_pod_cmd = [ + "kubectl", + "get", + "pods", + "-n", + "spark", + "-l", + "app.kubernetes.io/name=spark,app.kubernetes.io/component=master", + "-o", + "jsonpath='{.items[0].metadata.name}'", + ] + + spark_pod_result = subprocess.run(spark_pod_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + print(f"Connector Registry | spark_pod_result: {spark_pod_result}") + + if spark_pod_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to get the spark pod", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + spark_pod = spark_pod_result.stdout.decode("utf-8").replace("'", "") + print(f"Connector Registry | spark_pod: {spark_pod}") + + ## copy the connector to the spark pod under /data/connectors/{source} + source_path = os.path.join(self.extraction_path, connector_source) + copy_cmd = [ + "kubectl", + "cp", + f"{source_path}", + f"spark/{spark_pod}:/data/connectors/{connector_source}", + ] + + copy_result = subprocess.run(copy_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | copy_result: {copy_result}") + if copy_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to copy the connector to spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + if self.metadata['metadata']['technology'] == "python": + pip_install_cmd = [ + "kubectl", + "exec", + f"pod/{spark_pod}", + "-n", + "spark", + "--", + "bash", + "-c", + f"pip install -r /data/connectors/{connector_source}/requirements.txt", + ] + + pip_install_result = subprocess.run(pip_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | pip_install_result: {pip_install_result}") + if pip_install_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to install the requirements on spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + return RegistryResponse( + status="success", + message="connector copied to spark successfully", + statusCode=status.HTTP_200_OK, + ) + +class ExtractionUtil: + def extract_gz(tar_path, extract_path): + with tarfile.open(tar_path, "r:*") as tar: + tar.extractall(path=extract_path) + + def extract_zip(tar_path, extract_path): + with zipfile.ZipFile(tar_path, "r") as zip_ref: + zip_ref.extractall(path=extract_path) + + # Method to extract the compressed files + def extract(file, extract_out_path, ext) -> RegistryResponse: + extraction_function = ExtractionUtil.extract_gz + + compression_types = { + "zip": ExtractionUtil.extract_zip, + } + + try: + print( + f"Connector Registry | Extracting {file} to {extract_out_path} of {ext} file type" + ) + + if ext in compression_types: + extraction_function = compression_types.get(ext) + + extraction_function(file, extract_out_path) + print(f"Connector Registry | Extraction complete for {file}") + return RegistryResponse( + status="success", + message="Extraction Successfully Completed", + statusCode=status.HTTP_200_OK, + ) + except (tarfile.TarError, zipfile.BadZipFile, OSError) as e: + print( + f"Connector Registry | An error occurred while extracting the file: {e}" + ) + return RegistryResponse( + status="failure", + message="Failed to Extract the File", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + except Exception as e: + print(f"Connector Registry | An unexpected error occurred: {e}") + return RegistryResponse( + status="failure", + message="Failed to Extract the File", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py new file mode 100644 index 00000000..74a0a6fe --- /dev/null +++ b/command-service/src/command/dataset_command.py @@ -0,0 +1,106 @@ +import json +import time + +from dacite import from_dict +from command.icommand import ICommand +from config import Config +from model.data_models import CommandPayload, DatasetStatusType +from model.db_models import DatasetsLive +from model.telemetry_models import Audit, Object, Property, Transition +from service.db_service import DatabaseService +from service.http_service import HttpService +from service.telemetry_service import TelemetryService + +class DatasetCommand(ICommand): + def __init__( + self, + db_service: DatabaseService, + telemetry_service: TelemetryService, + http_service: HttpService, + config: Config, + ): + self.db_service = db_service + self.telemetry_service = telemetry_service + self.http_service = http_service + self.config = config + self.http_service = http_service + self.config_service_host = self.config.find("config_service.host") + self.config_service_port = self.config.find("config_service.port") + self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/datasets/v1/export" + + def _get_draft_dataset_record(self, dataset_id): + query = f""" + SELECT "type", MAX(version) AS max_version FROM datasets_draft WHERE dataset_id = %s GROUP BY 1 + """ + dataset_record = self.db_service.execute_select_one(sql=query, params=(dataset_id,)) + if dataset_record is not None: + return dataset_record + return None + + def _get_draft_dataset(self, dataset_id): + query = f""" + SELECT * FROM datasets_draft + WHERE dataset_id = %s AND (status = %s OR status = %s ) AND version = (SELECT MAX(version) + FROM datasets_draft WHERE dataset_id = %s AND (status = %s OR status = %s )) + """ + params = (dataset_id, DatasetStatusType.Publish.name, DatasetStatusType.ReadyToPublish.name, + dataset_id, DatasetStatusType.Publish.name, DatasetStatusType.ReadyToPublish.name,) + dataset_record = self.db_service.execute_select_one(sql=query, params=params) + if dataset_record is not None: + return dataset_record + return None + + def _check_for_live_record(self, dataset_id): + query = f""" + SELECT * FROM datasets WHERE dataset_id = %s AND status = %s + """ + params = (dataset_id, DatasetStatusType.Live.name, ) + result = self.db_service.execute_select_one(sql=query, params=params) + live_dataset = dict() + if result is not None: + live_dataset = from_dict(data_class=DatasetsLive, data=result) + data_version = live_dataset.data_version + 1 + return live_dataset, data_version + return None, None + + def audit_live_dataset(self, command_payload: CommandPayload, ts: int): + dataset_id = command_payload.dataset_id + dataset_record, data_version = self._check_for_live_record(dataset_id) + export_dataset = self.http_service.post( + url=self.base_url, + body=json.dumps({"dataset_id": dataset_id}), + headers={"Content-Type": "application/json"}, + ) + if export_dataset.status == 200: + result = json.loads(export_dataset.body) + object_ = Object( + dataset_id, dataset_record.type, dataset_record.data_version + ) + live_dataset_property = Property("dataset:export", result["result"], "") + draft_property = Property( + "draft-dataset:status", + DatasetStatusType.ReadyToPublish.name, + DatasetStatusType.Live.name, + ) + dataset_property = Property( + "dataset:status", + DatasetStatusType.Live.name, + DatasetStatusType.Live.name, + ) + transition = Transition(duration=int(time.time() - ts)) + edata = Audit( + props=[ + draft_property, + dataset_property, + live_dataset_property, + ], + transition=transition, + ) + self.telemetry_service.audit(object_=object_, edata=edata) + return True + else: + print( + "Failed to get dataset configurations from export API, dataset_id: ", + dataset_id, + ) + return False diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py new file mode 100644 index 00000000..89db3392 --- /dev/null +++ b/command-service/src/command/db_command.py @@ -0,0 +1,429 @@ +import json +import time +from datetime import datetime as dt +from dacite import from_dict +from command.dataset_command import DatasetCommand +from command.icommand import ICommand +from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from model.db_models import DatasetsDraft, DatasetConnectorConfigDraft, DatasourcesDraft, DatasetTransformationsDraft +from service.db_service import DatabaseService + +class DBCommand(ICommand): + + def __init__(self, db_service: DatabaseService, dataset_command: DatasetCommand): + self.db_service = db_service + self.dataset_command = dataset_command + + def execute(self, command_payload: CommandPayload, action: Action): + + if action == Action.MAKE_DATASET_LIVE.name: + print( + f"Invoking MAKE_DATASET_LIVE command for dataset_id {command_payload.dataset_id}..." + ) + result = self._change_dataset_to_active(command_payload) + print(f"Result from MAKE_DATASET_ACTIVE {result}...") + return result + else: + return ActionResponse( + status="ERROR", status_code=404, error_message="INVALID_ACTION" + ) + + def _change_dataset_to_active(self, command_payload: CommandPayload): + + dataset_id = command_payload.dataset_id + live_dataset, data_version = self.dataset_command._check_for_live_record( + dataset_id + ) + if live_dataset is not None: + self.dataset_command.audit_live_dataset( + command_payload, int(time.time() * 1000) + ) + + draft_dataset_record = self.dataset_command._get_draft_dataset(dataset_id) + + draft_dataset_id = self._insert_dataset_record( + dataset_id, data_version, live_dataset, draft_dataset_record + ) + if draft_dataset_id: + self._insert_datasource_record(dataset_id, draft_dataset_id) + self._insert_connector_instances(dataset_id, draft_dataset_record) + self._insert_dataset_transformations(dataset_id, draft_dataset_record) + self._delete_draft_dataset(dataset_id, draft_dataset_id) + return ActionResponse(status="OK", status_code=200) + else: + return ActionResponse( + status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" + ) + + def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_dataset_record): + + if draft_dataset_record is None: + return None + + draft_dataset = from_dict(data_class = DatasetsDraft, data = draft_dataset_record) + draft_dataset_id = draft_dataset.id + current_timestamp = dt.now() + params = ( + dataset_id, + dataset_id, + draft_dataset.type, + draft_dataset.name, + json.dumps(draft_dataset.extraction_config).replace("'", "''"), + json.dumps(draft_dataset.validation_config).replace("'", "''"), + json.dumps(draft_dataset.dedup_config).replace("'", "''"), + json.dumps(draft_dataset.denorm_config).replace("'", "''"), + json.dumps(draft_dataset.data_schema).replace("'", "''"), + json.dumps(draft_dataset.router_config).replace("'", "''"), + json.dumps(draft_dataset.dataset_config).replace("'", "''"), + DatasetStatusType.Live.name, + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}") if draft_dataset.tags is not None else json.dumps({}), + draft_dataset.api_version, + draft_dataset.version, + json.dumps(draft_dataset.sample_data).replace("'", "''"), + draft_dataset.entry_topic, + draft_dataset.created_by, + draft_dataset.updated_by, + current_timestamp, + current_timestamp, + current_timestamp, + + draft_dataset.name, + json.dumps(draft_dataset.extraction_config).replace("'", "''"), + json.dumps(draft_dataset.validation_config).replace("'", "''"), + json.dumps(draft_dataset.dedup_config).replace("'", "''"), + json.dumps(draft_dataset.denorm_config).replace("'", "''"), + json.dumps(draft_dataset.data_schema).replace("'", "''"), + json.dumps(draft_dataset.router_config).replace("'", "''"), + json.dumps(draft_dataset.dataset_config).replace("'", "''"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}") if draft_dataset.tags is not None else json.dumps({}), + data_version if live_dataset is not None else 1, + draft_dataset.api_version, + draft_dataset.version, + json.dumps(draft_dataset.sample_data).replace("'", "''"), + draft_dataset.entry_topic, + draft_dataset.updated_by, + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) + insert_query = f""" + INSERT INTO datasets(id, dataset_id, "type", name, extraction_config, validation_config, dedup_config, + denorm_config, data_schema, router_config, dataset_config, status, tags, data_version, api_version, + version, sample_data, entry_topic, created_by, updated_by, created_date, updated_date, published_date) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + 1, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s + ) + ON CONFLICT (id) DO UPDATE + SET name = %s, + extraction_config = %s, + validation_config = %s, + dedup_config = %s, + denorm_config = %s, + data_schema = %s, + router_config = %s, + dataset_config = %s, + tags = %s, + data_version = %s, + api_version = %s, + version = %s, + sample_data = %s, + entry_topic = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; + """ + self.db_service.execute_upsert(insert_query, params) + print(f"Dataset {dataset_id} record inserted successfully...") + return draft_dataset_id + + def _insert_datasource_record(self, dataset_id, draft_dataset_id): + + result = {} + draft_datasource_record = self.db_service.execute_select_all( + sql=f"SELECT * FROM datasources_draft WHERE dataset_id = %s", + params=(draft_dataset_id,) + ) + if draft_datasource_record is None: + return result + for record in draft_datasource_record: + draft_datasource = from_dict(data_class=DatasourcesDraft, data=record) + current_timestamp = dt.now() + params = ( + draft_datasource.id, + draft_datasource.datasource, + dataset_id, + draft_datasource.datasource_ref, + json.dumps(draft_datasource.ingestion_spec), + draft_datasource.type, + json.dumps(draft_datasource.retention_period).replace("'", "''"), + json.dumps(draft_datasource.archival_policy).replace("'", "''"), + json.dumps(draft_datasource.purge_policy).replace("'", "''"), + json.dumps(draft_datasource.backup_config).replace("'", "''"), + DatasetStatusType.Live.name, + draft_datasource.created_by, + draft_datasource.updated_by, + current_timestamp, + current_timestamp, + current_timestamp, + json.dumps(draft_datasource.metadata).replace("'", "''"), + + draft_datasource.datasource, + json.dumps(draft_datasource.ingestion_spec), + draft_datasource.type, + json.dumps(draft_datasource.retention_period).replace("'", "''"), + json.dumps(draft_datasource.archival_policy).replace("'", "''"), + json.dumps(draft_datasource.purge_policy).replace("'", "''"), + json.dumps(draft_datasource.backup_config).replace("'", "''"), + draft_datasource.updated_by, + current_timestamp, + current_timestamp, + json.dumps(draft_datasource.metadata).replace("'", "''"), + DatasetStatusType.Live.name, + ) + insert_query = f""" + INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, + archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, + updated_date, published_date, metadata) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s + ) + ON CONFLICT (id) DO UPDATE + SET datasource_ref = %s, + ingestion_spec = %s, + type = %s, + retention_period = %s, + archival_policy = %s, + purge_policy = %s, + backup_config = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + metadata = %s, + status = %s; + """ + result = self.db_service.execute_upsert(sql=insert_query, params=params) + print( + f"Datasource {draft_datasource.id} record inserted successfully..." + ) + return result + + def _insert_connector_instances(self, dataset_id, draft_dataset_record): + emptyJson = {} + result = {} + draft_connectors_config_record = draft_dataset_record.get('connectors_config') + if draft_connectors_config_record is None: + return result + + for record in draft_connectors_config_record: + connector_config = from_dict( + data_class = DatasetConnectorConfigDraft, data = record + ) + current_timestamp = dt.now() + if connector_config.version == 'v2': + params = ( + connector_config.id, + dataset_id, + connector_config.connector_id, + json.dumps(connector_config.connector_config).replace("'", "''"), + json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.data_format, + DatasetStatusType.Live.name, + json.dumps(emptyJson), + json.dumps(emptyJson), + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + + json.dumps(connector_config.connector_config).replace("'", "''"), + json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.data_format, + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) + insert_query = f""" + INSERT INTO connector_instances(id, dataset_id, connector_id, connector_config, operations_config, + data_format, status, connector_state, connector_stats, created_by, updated_by, created_date, + updated_date, published_date) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s + ) + ON CONFLICT (id) DO UPDATE + SET connector_config = %s, + operations_config = %s, + data_format = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; + """ + result = self.db_service.execute_upsert(sql=insert_query, params=params) + print( + f"Connector[v2] Instance record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." + ) + else: + params = ( + connector_config.id, + dataset_id, + connector_config.connector_id, + json.dumps(connector_config.connector_config).replace("'", "''"), + DatasetStatusType.Live.name, + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + + json.dumps(connector_config.connector_config).replace("'", "''"), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) + insert_query = f""" + INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, + status, created_by, updated_by, created_date, updated_date, published_date) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s + ) + ON CONFLICT (id) DO UPDATE + SET connector_config = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; + """ + result = self.db_service.execute_upsert(sql=insert_query, params=params) + print( + f"Connector[v1] record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." + ) + + return result + + def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): + + draft_dataset_transformations_record = draft_dataset_record.get('transformations_config') + result = {} + current_timestamp = dt.now() + # Delete existing transformations + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations where dataset_id = %s""", params=(dataset_id,)) + print(f"Dataset Transformation for {dataset_id} are deleted successfully...") + + if draft_dataset_transformations_record is None: + return result + + for record in draft_dataset_transformations_record: + transformation = from_dict( + data_class=DatasetTransformationsDraft, data=record + ) + params = ( + dataset_id + '_' + transformation.field_key, + dataset_id, + transformation.field_key, + json.dumps(transformation.transformation_function).replace("'", "''"), + DatasetStatusType.Live.name, + transformation.mode, + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + ) + insert_query = f""" + INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, + status, mode, created_by, updated_by, created_date, updated_date, published_date) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s + ) + """ + result = self.db_service.execute_upsert(sql=insert_query, params=params) + print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") + return result + + def _delete_draft_dataset(self, dataset_id, draft_dataset_id): + + self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft source config/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) + print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py new file mode 100644 index 00000000..f43222b4 --- /dev/null +++ b/command-service/src/command/druid_command.py @@ -0,0 +1,53 @@ +import json + +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.db_service import DatabaseService +from service.http_service import HttpService + + +class DruidCommand(ICommand): + + def __init__( + self, config: Config, db_service: DatabaseService, http_service: HttpService + ): + self.config = config + self.db_service = db_service + self.http_service = http_service + router_host = self.config.find("druid.router_host") + router_post = self.config.find("druid.router_port") + self.supervisor_endpoint = self.config.find("druid.supervisor_endpoint") + self.router_url = f"{router_host}:{router_post}/druid" + + def execute(self, command_payload: CommandPayload, action: Action): + if action == Action.SUBMIT_INGESTION_TASKS.name: + response = self._submit_ingestion_task(command_payload.dataset_id) + return response + + def _submit_ingestion_task(self, dataset_id): + datasources_records = self.db_service.execute_select_all( + sql=f"SELECT dso.*, dt.type as dataset_type FROM datasources dso, datasets dt WHERE dso.dataset_id = %s AND dso.dataset_id = dt.id", + params=(dataset_id,) + ) + if datasources_records is not None: + print( + f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." + ) + for record in datasources_records: + if record["dataset_type"] == "event" and record["type"] == "druid": + print(f"Submitting ingestion task for datasource ...") + ingestion_spec = json.dumps(record["ingestion_spec"]) + response = self.http_service.post( + url=f"{self.router_url}/{self.supervisor_endpoint}", + body=ingestion_spec, + headers={"Content-Type": "application/json"}, + ) + return ActionResponse(status="OK", status_code=200) + else: + print( + f"Dataset ID {dataset_id} not found for druid ingestion task submit..." + ) + return ActionResponse( + status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" + ) diff --git a/command-service/src/command/flink_command.py b/command-service/src/command/flink_command.py index c3397923..293f54dc 100644 --- a/command-service/src/command/flink_command.py +++ b/command-service/src/command/flink_command.py @@ -1,9 +1,11 @@ +import logging +import subprocess + from command.icommand import ICommand -from model.data_models import CommandPayload, Action, ActionResponse from config import Config +from model.data_models import Action, ActionResponse, CommandPayload from service.http_service import HttpService -import subprocess -import logging + class FlinkCommand(ICommand): @@ -14,29 +16,51 @@ def __init__(self, config: Config, http_service: HttpService): def execute(self, command_payload: CommandPayload, action: Action): result = None - print(f"Invoking RESTART_PIPELINE_JOBS command...") - result = self._restart_jobs() + if action == Action.START_PIPELINE_JOBS.name: + print( + f"Invoking START_PIPELINE_JOBS command for dataset_id {command_payload.dataset_id}..." + ) + result = self._restart_jobs() return result def _restart_jobs(self): return self._install_flink_jobs() - + + def _restart_pods(self, release_name, namespace, job_name): + restart_cmd = f"kubectl delete pods --selector app=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app=flink,component={release_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, release_name=release_name + ) + # Run the helm command + helm_install_result = subprocess.run( + restart_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + if helm_install_result.returncode == 0: + print(f"Job {job_name} re-deployment succeeded...") + return True + else: + print( + f"Error re-installing job {job_name}: {helm_install_result.stderr.decode()}" + ) + return False + def _install_flink_jobs(self): - namespace = self.config.find("flink.namespace") result = ActionResponse(status="OK", status_code=200) - flink_jobs = self.config.find("flink.jobs") + namespace = self.config.find("flink.namespace") for job in flink_jobs: release_name = job["release_name"] job_name = job["name"] - restart_cmd = f"kubectl delete pods --selector app=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app=flink,component={release_name}-taskmanager --namespace {namespace}".format(namespace=namespace, release_name=release_name) - # Run the helm command - helm_install_result = subprocess.run(restart_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,) - - if helm_install_result.returncode == 0: - print(f"Job {job_name} re-deployment succeeded...") - else: - print(f"Error re-installing job {job_name}: {helm_install_result.stderr.decode()}") - result = ActionResponse(status="ERROR", status_code=500, error_message="FLINK_HELM_INSTALLATION_EXCEPTION") - + # Restart pods + status = self._restart_pods( + release_name=release_name, namespace=namespace, job_name=job_name + ) + if not status: + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + ) return result diff --git a/command-service/src/command/icommand.py b/command-service/src/command/icommand.py index 6271b933..66938fd0 100644 --- a/command-service/src/command/icommand.py +++ b/command-service/src/command/icommand.py @@ -1,7 +1,7 @@ from model.data_models import Action, CommandPayload + class ICommand: def execute(self, command_payload: CommandPayload, action: Action): pass - diff --git a/command-service/src/command/telemetry_command.py b/command-service/src/command/telemetry_command.py new file mode 100644 index 00000000..d16d7abb --- /dev/null +++ b/command-service/src/command/telemetry_command.py @@ -0,0 +1,59 @@ +import time + +from command.dataset_command import DatasetCommand +from command.icommand import ICommand +from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from model.telemetry_models import Audit, Object, Property, Transition +from service.telemetry_service import TelemetryService + + +class TelemetryCommand(ICommand): + def __init__( + self, telemetry_service: TelemetryService, dataset_command: DatasetCommand + ): + self.dataset_command = dataset_command + self.telemetry_service = telemetry_service + + def execute(self, command_payload: CommandPayload, action: Action, ts: int): + if action == Action.CREATE_AUDIT_EVENT.name: + print( + f"Invoking CREATE_AUDIT_EVENT command for dataset_id {command_payload.dataset_id}..." + ) + object_, audit, error = self._create_audit_event(command_payload, ts) + if error: + return error + self.telemetry_service.audit(object_=object_, edata=audit) + return ActionResponse(status="OK", status_code=200) + + def _create_audit_event(self, command_payload: CommandPayload, ts: int): + dataset_id = command_payload.dataset_id + dataset_record = self.dataset_command._get_draft_dataset_record(dataset_id) + if dataset_record: + object_ = Object( + dataset_id, dataset_record["type"], dataset_record["max_version"] + ) + draft_property = Property( + "draft-dataset:status", + DatasetStatusType.ReadyToPublish.name, + DatasetStatusType.Live.name, + ) + dataset_property = Property( + "dataset:status", + DatasetStatusType.ReadyToPublish.name, + DatasetStatusType.Live.name, + ) + transition = Transition(duration=int(time.time() - ts)) + edata = Audit( + props=[draft_property, dataset_property], transition=transition + ) + return object_, edata, None + else: + return ( + None, + None, + ActionResponse( + status="ERROR", + status_code=404, + error_message="DATASET_ID_NOT_FOUND", + ), + ) diff --git a/command-service/src/config/__init__.py b/command-service/src/config/__init__.py index 3558f420..e0db2235 100644 --- a/command-service/src/config/__init__.py +++ b/command-service/src/config/__init__.py @@ -1 +1,2 @@ -from .config import Config \ No newline at end of file +# autoflake: skip_file +from .config import Config diff --git a/command-service/src/config/config.py b/command-service/src/config/config.py index d931dda4..e96475a2 100644 --- a/command-service/src/config/config.py +++ b/command-service/src/config/config.py @@ -1,13 +1,22 @@ -import yaml -from functools import reduce import operator +import os +from functools import reduce + +import yaml + +# import json +# from yaml.loader import SafeLoader -class Config(): +class Config: def __init__(self): - with open('config/service_config.yml') as config_file: + config_file = os.getenv("CONFIG_PATH", "config") + with open(os.path.join(config_file, "service_config.yml")) as config_file: self.config = yaml.safe_load(config_file) def find(self, path): element_value = reduce(operator.getitem, path.split("."), self.config) - return element_value \ No newline at end of file + # if (isinstance(element_value, list)): + # element_value = json.dumps(element_value) + # print("element_value = {0}".format(element_value)) + return element_value diff --git a/command-service/src/config/pii_rules.yml b/command-service/src/config/pii_rules.yml new file mode 100644 index 00000000..6947f9ad --- /dev/null +++ b/command-service/src/config/pii_rules.yml @@ -0,0 +1,118 @@ +keys: + address: + rule: '\baddress\b|\blocation\b|\bloc\b|\baddr\b|\badd\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m001 + financial: + rule: '\bcredit\b|\bdebit\b|\baccount\b|\btransaction\b|\btxn\b|\biban\b|\bswift\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m002 + id: + rule: '\bid\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m003 + internet: + rule: '\bemail\b|\bip\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m004 + phone: + rule: '\bphone\b|\bnumber\b|\bno\b|\bno\.\b|\bnum\b|\bmobile\b|\bcontact\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m005 + name: + rule: '\bname\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m006 +values: + address: + india: + rule: '(?:\bAN\b|\bAP\b|\bAD\b|\bAR\b|\bAS\b|\bBH\b|\bCH\b|\bCT\b|\bDN\b|\bDL\b|\bGA\b|\bGJ\b|\bHR\b|\bHP\b|\bJK\b|\bJH\b|\bKA\b|\bKL\b|\bLD\b|\bMP\b|\bMH\b|\bMN\b|\bME\b|\bMI\b|\bNL\b|\bOR\b|\bPY\b|\bPB\b|\bRJ\b|\bSK\b|\bTN\b|\bTS\b|\bTR\b|\bUP\b|\bUT\b|\bWB\b)' + locale: 'India' + code: 'en' + resourceKey: pii.descriptions.m007 + us: + rule: '\d{1,4} [\w\s]{1,20}(?:\bstreet\b|\bst\b|\bavenue\b|\bave\b|\broad\b|\brd\b|\bhighway\b|\bhwy\b|\bsquare\b|\bsq\b|\btrail\b|\btrl\b|\bdrive\b|\bdr\b|\bcourt\b|\bct\b|\bpark\b|\bparkway\b|\bpkwy\b|\bcircle\b|\bcir\b|\bboulevard\b|\bblvd\b)\W?(?=\s|$)' + locale: 'USA' + code: 'en' + resourceKey: pii.descriptions.m008 + financial: + credit_card: + rule: '((?:(?:\\d{4}[- ]?){3}\\d{4}|\\d{15,16}))(?![\\d])' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m009 + iban_number: + rule: '[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z\d]?){0,16}' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m010 + id: + aadhaar: + rule: '\b\d{4}[ -]\d{4}[ -]\d{4}' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m011 + pan: + rule: '\b[A-Z]{4}\d{4}[A-Z]' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m012 + ssn: + rule: '(?:\d{3}-\d{2}-\d{4})' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m013 + internet: + email: + rule: '(?i)([A-Za-z0-9!#$%&*+\/=?^_{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m014 + ipv4: + rule: &IPv4 '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m015 + ipv6: + rule: &IPv6 '\s*(?!.*::.*::)(?:(?!:)|:(?=:))(?:[0-9a-f]{0,4}(?:(?<=::)|(? None: + self.node_query_response_time = Gauge( + name="node_query_response_time", + documentation="The average response time for database queries", + labelnames=["entity", "id", "endpoint", "datasetId", "status"], + registry=registry, + ) + self.total_api_calls = Counter( + name="node_total_api_calls", + documentation="The total number of API calls made", + labelnames=["entity", "id", "endpoint", "datasetId"], + registry=registry, + ) + self.failed_api_calls = Counter( + name="node_failed_api_calls", + documentation="The total number of failed API calls made", + labelnames=["entity", "id", "endpoint", "datasetId", "status"], + registry=registry, + ) + self.success_api_calls = Counter( + name="node_success_api_calls", + documentation="The total number of successful API calls made", + labelnames=["entity", "id", "endpoint", "datasetId", "status"], + registry=registry, + ) + + def queryResponseTimeMetric(self): + return self.node_query_response_time + + def totalApiCallsMetric(self): + return self.total_api_calls + + def failedApiCallsMetric(self): + return self.failed_api_calls + + def successApiCallsMetric(self): + return self.success_api_calls diff --git a/command-service/src/model/__init__.py b/command-service/src/model/__init__.py index d336e01c..5b591820 100644 --- a/command-service/src/model/__init__.py +++ b/command-service/src/model/__init__.py @@ -1 +1,10 @@ -from .data_models import Command, Action, CommandPayload, Request, ResponseParams, Result, HttpResponse \ No newline at end of file +# autoflake: skip_file +from .data_models import ( + Action, + Command, + CommandPayload, + HttpResponse, + Request, + ResponseParams, + Result, +) diff --git a/command-service/src/model/data_models.py b/command-service/src/model/data_models.py index 910afca1..f248e6ba 100644 --- a/command-service/src/model/data_models.py +++ b/command-service/src/model/data_models.py @@ -1,34 +1,63 @@ +import time from dataclasses import dataclass -from dataclasses_json import dataclass_json from enum import Enum +from typing import List + +from dataclasses_json import dataclass_json + class Command(Enum): + PUBLISH_DATASET = "PUBLISH_DATASET" RESTART_PIPELINE = "RESTART_PIPELINE" + RESTART_CONNECTORS = "RESTART_CONNECTORS" class Action(Enum): - RESTART_PIPELINE_JOBS = "RESTART_PIPELINE_JOBS" + SUBMIT_INGESTION_TASKS = "SUBMIT_INGESTION_TASKS" + MAKE_DATASET_LIVE = "MAKE_DATASET_LIVE" + MAKE_DATASOURCE_ACTIVE = "MAKE_DATASOURCE_ACTIVE" + START_PIPELINE_JOBS = "START_PIPELINE_JOBS" + CREATE_ALERT_METRIC = "CREATE_ALERT_METRIC" + DEPLOY_CONNECTORS = "DEPLOY_CONNECTORS" + CREATE_AUDIT_EVENT = "CREATE_AUDIT_EVENT" + + +class DatasetStatusType(Enum): + Draft = "Draft" + Publish = "Publish" + ReadyToPublish = "ReadyToPublish" + Live = "Live" + Retired = "Retired" + Purged = "Purged" + @dataclass_json @dataclass -class CommandPayload(): +class CommandPayload: + dataset_id: str command: Command + @dataclass_json @dataclass class Request: data: CommandPayload id: str + @dataclass_json @dataclass class ResponseParams: status: str + resmsgid: str | None = None + @dataclass_json @dataclass class Result: + dataset_id: str message: str + @dataclass_json @dataclass class Response: @@ -39,11 +68,13 @@ class Response: ts: str | None = None params: ResponseParams | None = None + @dataclass_json @dataclass class HttpResponse: status: int - body: str + body: str + @dataclass class ActionResponse: @@ -51,3 +82,73 @@ class ActionResponse: status_code: int error_message: str = None + +@dataclass +class DatasetRequest: + id: str + dataset_id: str + data: List[dict] + + +@dataclass +class PIIReason: + code: str + resourceKey: str + region: str | None = None + score: float | None = None + + +@dataclass +class PIIError: + errorCode: int + errorMsg: str + errorTrace: str + + +@dataclass +class PIIResult: + field: str + type: str + score: float + reason: List[PIIReason] + + +@dataclass +class DatasetResponse: + id: str + response_code: str + status_code: int + result: List[PIIResult] | PIIError + ts: str | None = None + params: ResponseParams | None = None + + +class PIIModel: + def __init__(self): + pass + + def detect_pii(self, entity, field, value) -> PIIResult: + return PIIResult() + + def detect_entity(self, entity, value) -> List[PIIReason]: + return [PIIReason()] + + def detect_entity_in_fieldname(self, entity, value) -> List[PIIReason]: + return [PIIReason()] + + +class ParamsModels: + def __init__(self): + self.status = "" + self.msgid = "" + + def to_dict(self): + return {"status": self.status, "msgid": self.msgid} + + +class ConnectorResponseModel: + def __init__(self, id: str, ver: str): + self.id = id + self.ver = ver + self.ts = str(time.time() * 1000) + self.params = ParamsModels() diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py new file mode 100644 index 00000000..eaeea86d --- /dev/null +++ b/command-service/src/model/db_models.py @@ -0,0 +1,127 @@ +from dataclasses import dataclass +from datetime import datetime +from dataclasses_json import dataclass_json + +@dataclass +class DatasetsLive: + id: str + dataset_id: str + data_version: int + type: str + name: str + validation_config: dict + extraction_config: dict + dedup_config: dict + data_schema: dict + router_config: dict + dataset_config: dict + version: int + status: str + created_by: str + created_date: datetime + tags: list[str] | None = None + updated_by: str | None = None + updated_date: datetime | None = None + denorm_config: dict | None = None + published_date: datetime | None = None + + +@dataclass +class DatasetsDraft: + id: str + dataset_id: str + version: int + type: str + name: str + validation_config: dict + extraction_config: dict + dedup_config: dict + data_schema: dict + router_config: dict + dataset_config: dict + status: str + api_version: str + entry_topic: str + created_by: str + created_date: datetime + sample_data: dict | None = None + transformations_config: list[dict] | None = None + connectors_config: list[dict] | None = None + denorm_config: dict | None = None + tags: list[str] | None = None + updated_by: str | None = None + updated_date: datetime | None = None + published_date: datetime | None = None + + +@dataclass +class DatasourcesDraft: + id: str + datasource: str + dataset_id: str + ingestion_spec: dict + type: str + datasource_ref: str + status: str + created_by: str + created_date: datetime + retention_period: dict | None = None + archival_policy: dict | None = None + purge_policy: dict | None = None + backup_config: dict | None = None + metadata: dict | None = None + updated_by: str | None = None + updated_date: datetime | None = None + published_date: datetime | None = None + + +@dataclass +class DatasetConnectorConfigDraft: + id: str + connector_id: str + connector_config: str | dict + version: str + operations_config: dict | None = None + data_format: str | None = 'json' + + +@dataclass +class DatasetTransformationsDraft: + field_key: str + transformation_function: dict + mode: str + + +@dataclass_json +@dataclass +class ConnectorRegsitryv2: + id: str + name: str + type: str + category: str + version: str + description: str + technology: str + runtime: str + licence: str + owner: str + iconurl: str + status: str + source_url: str + source: str + created_by: str + created_date: str + updated_date: str + ui_spec: dict | None = None + updated_by: str | None = None + livedate: datetime | None = None + + +@dataclass +class ConnectorInstance: + id: str + connector_id: str + operations_config: dict + connector_runtime: str + connector_source: dict + technology: str \ No newline at end of file diff --git a/command-service/src/model/telemetry_models.py b/command-service/src/model/telemetry_models.py new file mode 100644 index 00000000..b2fb0d43 --- /dev/null +++ b/command-service/src/model/telemetry_models.py @@ -0,0 +1,75 @@ +import time +from dataclasses import dataclass +from uuid import uuid4 + +from dataclasses_json import dataclass_json + +from .data_models import DatasetStatusType + + +@dataclass_json +@dataclass +class Actor: + id: str = "SYSTEM" + type: str = "User" + + +@dataclass_json +@dataclass +class Pdata: + id: str + ver: str | None = "1.0.0" + + +@dataclass_json +@dataclass +class Context: + pdata: Pdata | None + env: str + sid: str = uuid4() + + +@dataclass_json +@dataclass +class Object: + id: str + type: str + ver: str | None + + +@dataclass_json +@dataclass +class Property: + property: str + ov: str + nv: str + + +@dataclass_json +@dataclass +class Transition: + duration: str + fromState: str | None = DatasetStatusType.ReadyToPublish.name + toState: str | None = DatasetStatusType.Live.name + timeunit: str = "milliseconds" + + +@dataclass_json +@dataclass +class Audit: + props: list[Property] + transition: Transition + action: str = "dataset:publish" + + +@dataclass_json +@dataclass +class Telemetry: + actor: Actor + context: Context + object: Object | None + edata: Audit | None + eid: str = "AUDIT" + ets: int = time.time() * 1000 + ver: str = "1.0.0" + mid: str = uuid4() diff --git a/command-service/src/routes.py b/command-service/src/routes.py index 6365fd04..d94ce922 100644 --- a/command-service/src/routes.py +++ b/command-service/src/routes.py @@ -1,30 +1,236 @@ -from fastapi import FastAPI, status -from command.command_executor import CommandExecutor -from model.data_models import Request, Response, Result, ActionResponse +import time +import uuid from datetime import datetime as dt +from typing import List + +from fastapi import FastAPI +from fastapi import Request as FastAPIRequest +from fastapi import Response, status +from fastapi.responses import JSONResponse, PlainTextResponse +from prometheus_client import CollectorRegistry, generate_latest + +from command.command_executor import CommandExecutor +from command.connector_registry import ConnectorRegistry +from metrics import Helper +from model.data_models import ( + ActionResponse, + ConnectorResponseModel, + DatasetRequest, + DatasetResponse, + PIIError, + PIIResult, + Request, + Response, + Result, +) +from service.detect_pii_service import DetectPIIService app = FastAPI() command_executor = CommandExecutor() +pii_service = DetectPIIService() +registry = CollectorRegistry() +helper = Helper(registry) -@app.post("/system/dataset/command") -async def restart_pipeline(request: Request): +system_dataset_endpoint = "/system/v1/dataset/command" + + +@app.post(system_dataset_endpoint) +async def publish_dataset(request: Request): + start_time = int(time.time() * 1000) data = request.data - result: ActionResponse = command_executor.execute_command(payload=data) + helper.onRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + ) + result: ActionResponse = command_executor.execute_command( + payload=data, ts=start_time + ) if result.status_code == status.HTTP_404_NOT_FOUND: - response = get_response_object(request_id=request.id, response_code="ERROR", - status_code=status.HTTP_404_NOT_FOUND, message=result.error_message) + helper.onFailedRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + status=404, + ) + response = get_response_object( + dataset_id=data.dataset_id, + request_id=request.id, + response_code="ERROR", + status_code=status.HTTP_404_NOT_FOUND, + message=result.error_message, + ) elif result.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR: - response = get_response_object(request_id=request.id, response_code="ERROR", - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, message=result.error_message) + helper.onFailedRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + status=500, + ) + response = get_response_object( + dataset_id=data.dataset_id, + request_id=request.id, + response_code="ERROR", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + message=result.error_message, + ) else: - response = get_response_object(request_id=request.id, response_code="OK", - status_code=status.HTTP_200_OK, message="RESTARTING_PIPELINE_SUCCESSFUL") + helper.onSuccessRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + ) + response = get_response_object( + dataset_id=data.dataset_id, + request_id=request.id, + response_code="OK", + status_code=status.HTTP_200_OK, + message="PUBLISH_DATASET_SUCCESSFUL", + ) return response -def get_response_object(request_id, response_code, status_code, message=None): - response = Response(id=request_id, - response_code=response_code, - status_code=status_code, - ts=dt.now().strftime("%Y-%m-%d %H:%M:%S"), - result=Result(message=message)) + +def get_response_object( + dataset_id, request_id, response_code, status_code, message=None +): + response = Response( + id=request_id, + response_code=response_code, + status_code=status_code, + ts=dt.now().strftime("%Y-%m-%d %H:%M:%S"), + result=Result(dataset_id=dataset_id, message=message), + ) return response + + +pii_endpoint = "/system/data/v1/analyze/pii" + + +@app.post(pii_endpoint) +def analyze_pii(request: DatasetRequest) -> DatasetResponse: + helper.onRequest( + entity="dataset", id=request.id, endpoint=pii_endpoint, dataset_id=None + ) + try: + event_data = request.data[0] + except Exception as err: + result: PIIError = { + "errorCode": 500, + "errorMsg": type(err), + "errorTrace": err.args, + } + else: + result: List[PIIResult] | PIIError = pii_service.detect_pii_fields(event_data) + finally: + if type(result) == list: + helper.onSuccessRequest( + entity="dataset", id=request.id, endpoint=pii_endpoint, dataset_id=None + ) + response_code = "OK" + status_code = 200 + else: + helper.onFailedRequest( + entity="dataset", + id=request.id, + endpoint=pii_endpoint, + dataset_id=None, + status=500, + ) + response_code = "INTERNAL_SERVER_ERROR" + status_code = 500 + response: DatasetResponse = { + "id": str(uuid.uuid4()), + "response_code": response_code, + "status_code": status_code, + "result": result, + "ts": str(time.time() * 1000), + "params": {"status": "ACTIVE"}, + } + return response + + +@app.get("/metrics", response_class=PlainTextResponse) +def metrics(): + data = generate_latest(registry=registry) + return data + + +connector_registry_endpoint = "/connector/v1/register" + + +@app.post(connector_registry_endpoint) +async def register_connector(req: FastAPIRequest): + response = ConnectorResponseModel(id="api.connector.registry", ver="1.0") + try: + data = await req.json() + response.params.msgid = data.get("params", {}).get("msgid", str(uuid.uuid4())) + + # Get the connector relative path in data + rel_path: str = data.get("relative_path", None) + + print( + f"Connector Registry | Received request to register connector: {rel_path}" + ) + + if not rel_path: + response.params.status = "Failure" + return JSONResponse( + { + "id": response.id, + "ver": response.ver, + "ts": response.ts, + "params": response.params.to_dict(), + "responseCode": status.HTTP_400_BAD_REQUEST, + "error": {"message": "connector relative path is missing."}, + }, + status_code=status.HTTP_400_BAD_REQUEST, + ) + + executor = ConnectorRegistry() + result = executor.register(rel_path) + response.params.status = result.status + + content = { + "id": response.id, + "ver": response.ver, + "ts": response.ts, + "params": response.params.to_dict(), + "responseCode": result.status, + } + + print(f"Connector Registry | Connector registration status: {content}") + + if result.status == "success": + content["message"] = "connector registered successfully." + content["connector_info"] = result.connector_info + else: + content["error"] = {"message": result.message} + + return JSONResponse(content=content, status_code=result.statusCode) + + except Exception as e: + print(f"Connector Registry | An error occurred while Calling API: {e}") + return JSONResponse( + content={ + "id": response.id, + "ver": response.ver, + "ts": response.ts, + "params": response.params.to_dict(), + "responseCode": status.HTTP_500_INTERNAL_SERVER_ERROR, + "error": {"message": str(e)}, + }, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +if __name__ == "__main__": + import uvicorn + + print("Starting server") + config = uvicorn.Config("routes:app", port=8000, log_level="info") + server = uvicorn.Server(config) + server.run() diff --git a/command-service/src/service/__init__.py b/command-service/src/service/__init__.py index 6ae50b6d..521c0d1d 100644 --- a/command-service/src/service/__init__.py +++ b/command-service/src/service/__init__.py @@ -1 +1,3 @@ -from .http_service import HttpService \ No newline at end of file +# autoflake: skip_file +from .db_service import DatabaseService +from .http_service import HttpService diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py new file mode 100644 index 00000000..ded56b91 --- /dev/null +++ b/command-service/src/service/db_service.py @@ -0,0 +1,73 @@ +import psycopg2 +import psycopg2.extras + +from typing import Callable +from tenacity import retry, stop_after_attempt, wait_exponential +from config import Config + +def reconnect(func: Callable): + + def wrapper(db_connection, *args, **kwargs): + tdecorator = retry(wait=wait_exponential(), stop=stop_after_attempt(3)) + decorated = tdecorator(func) + return decorated(db_connection, *args, **kwargs) + + return wrapper + + +class DatabaseService: + + def __init__(self): + self.config = Config() + + def connect(self): + # print("Connecting to postgresql db...") + db_host = self.config.find("postgres.db_host") + db_port = self.config.find("postgres.db_port") + db_user = self.config.find("postgres.db_user") + db_password = self.config.find("postgres.db_password") + database = self.config.find("postgres.database") + db_connection = psycopg2.connect( + database=database, + host=db_host, + port=db_port, + user=db_user, + password=db_password, + ) + db_connection.autocommit = True + return db_connection + + # @reconnect + def execute_select_one(self, sql, params): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql, params) + result = cursor.fetchone() + db_connection.close() + return result + + # @reconnect + def execute_select_all(self, sql, params): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql, params) + result = cursor.fetchall() + db_connection.close() + return result + + # @reconnect + def execute_upsert(self, sql, params): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql, params) + record_count = cursor.rowcount + db_connection.close() + # print(f"{record_count} inserted/updated successfully") + return record_count + +# @reconnect + def execute_delete(self, sql, params): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql, params) + db_connection.close() diff --git a/command-service/src/service/detect_pii_service.py b/command-service/src/service/detect_pii_service.py new file mode 100644 index 00000000..b827b624 --- /dev/null +++ b/command-service/src/service/detect_pii_service.py @@ -0,0 +1,33 @@ +from typing import List + +from model.data_models import PIIError, PIIResult +from service.re_pii_model import REPIIModel + + +class DetectPIIService: + def __init__(self) -> None: + self.model = REPIIModel() + + def detect_pii_fields(self, event_data: dict) -> List[PIIResult] | PIIError: + try: + results = [] + for field in list(event_data.keys()): + results += self.model.detect_pii( + "address", field, str(event_data[field]) + ) + results += self.model.detect_pii( + "financial", field, str(event_data[field]) + ) + results += self.model.detect_pii("id", field, str(event_data[field])) + results += self.model.detect_pii( + "internet", field, str(event_data[field]) + ) + results += self.model.detect_pii("phone", field, str(event_data[field])) + return results + except Exception as err: + pii_error: PIIError = { + "errorCode": 500, + "errorMsg": type(err), + "errorTrace": err.args, + } + return pii_error diff --git a/command-service/src/service/http_service.py b/command-service/src/service/http_service.py index 16059f2e..9d136631 100644 --- a/command-service/src/service/http_service.py +++ b/command-service/src/service/http_service.py @@ -1,28 +1,33 @@ -import urllib3 -from urllib3.exceptions import NewConnectionError, MaxRetryError import backoff +import urllib3 +from urllib3.exceptions import MaxRetryError, NewConnectionError from model.data_models import HttpResponse -class HttpService(): + +class HttpService: def __init__(self): self.http = urllib3.PoolManager(num_pools=5) - @backoff.on_exception(wait_gen=backoff.expo, - exception=(NewConnectionError, MaxRetryError), - max_tries=3) + @backoff.on_exception( + wait_gen=backoff.expo, + exception=(NewConnectionError, MaxRetryError), + max_tries=3, + ) def post(self, url: str, body=None, headers=None): - response = self.http.request('POST', url, body=body, headers=headers) - return HttpResponse(status = response.status, body = response.data.decode("utf-8")) - - @backoff.on_exception(wait_gen=backoff.expo, - exception=(NewConnectionError, MaxRetryError), - max_tries=3) + response = self.http.request("POST", url, body=body, headers=headers) + return HttpResponse(status=response.status, body=response.data.decode("utf-8")) + + @backoff.on_exception( + wait_gen=backoff.expo, + exception=(NewConnectionError, MaxRetryError), + max_tries=3, + ) def get(self, url: str): - response = self.http.request('GET', url) - return HttpResponse(status = response.status, body = response.data.decode("utf-8")) - + response = self.http.request("GET", url) + return HttpResponse(status=response.status, body=response.data.decode("utf-8")) + def delete(self, url: str): - response = self.http.request('DELETE', url) - return HttpResponse(status = response.status, body = response.data.decode("utf-8")) \ No newline at end of file + response = self.http.request("DELETE", url) + return HttpResponse(status=response.status, body=response.data.decode("utf-8")) diff --git a/command-service/src/service/re_pii_model.py b/command-service/src/service/re_pii_model.py new file mode 100644 index 00000000..52031946 --- /dev/null +++ b/command-service/src/service/re_pii_model.py @@ -0,0 +1,64 @@ +import re +from typing import List + +import yaml + +from model.data_models import PIIModel, PIIReason, PIIResult + + +class REPIIModel(PIIModel): + def __init__(self): + def join(loader, node): + seq = loader.construct_sequence(node) + return "".join([str(i) for i in seq]) + + yaml.add_constructor("!join", join) + with open("config/pii_rules.yml", "r") as f: + self.pii_rules = yaml.load(f, Loader=yaml.FullLoader) + + def detect_pii(self, entity, field, value) -> List[PIIResult]: + results = [] + reasons = [] + reasons += self.detect_entity(entity, value) + reasons += self.detect_entity_in_fieldname(entity, field) + if len(reasons) != 0: + score = 1 / len(reasons) + result: PIIResult = { + "field": field, + "type": entity, + "score": score, + "reason": reasons, + } + results.append(result) + return results + + def detect_entity(self, entity, value) -> List[PIIReason]: + matches = [] + for rule in list(self.pii_rules["values"][entity].keys()): + rule_matches = list( + re.findall(self.pii_rules["values"][entity][rule]["rule"], value) + ) + if len(rule_matches) != 0: + reason: PIIReason = { + "code": self.pii_rules["values"][entity][rule]["code"], + "resourceKey": self.pii_rules["values"][entity][rule][ + "resourceKey" + ], + "region": self.pii_rules["values"][entity][rule]["locale"], + "score": 1 / len(rule_matches), + } + matches.append(reason) + return matches + + def detect_entity_in_fieldname(self, entity, value) -> List[PIIReason]: + matches = [] + rule_matches = list(re.findall(self.pii_rules["keys"][entity]["rule"], value)) + if len(rule_matches) != 0: + reason: PIIReason = { + "code": self.pii_rules["keys"][entity]["code"], + "resourceKey": self.pii_rules["keys"][entity]["resourceKey"], + "region": self.pii_rules["keys"][entity]["locale"], + "score": 1 / len(rule_matches), + } + matches.append(reason) + return matches diff --git a/command-service/src/service/telemetry_service.py b/command-service/src/service/telemetry_service.py new file mode 100644 index 00000000..ebc9033d --- /dev/null +++ b/command-service/src/service/telemetry_service.py @@ -0,0 +1,38 @@ +import json +import os + +from kafka import KafkaProducer + +from config import Config +from model.telemetry_models import Actor, Audit, Context, Object, Pdata, Telemetry + + +class TelemetryService: + def __init__(self) -> None: + self.actor = Actor() + self.pdata = Pdata("{}.command.service".format(os.getenv("system_env", "dev"))) + self.context = Context(self.pdata, os.getenv("system_env", "dev")) + self.config_obj = Config() + try: + self.producer = KafkaProducer( + value_serializer=lambda v: json.dumps(v).encode("utf-8"), + bootstrap_servers=self.config_obj.find("kafka.brokers"), + ) + except Exception as e: + print("Error while connecting to kafka - ", e) + self.producer = None + self.topic = ( + os.getenv("system_env", "dev") + + "." + + self.config_obj.find("kafka.telemetry.topic") + ) + + def audit(self, object_: Object, edata: Audit): + event = Telemetry( + actor=self.actor, context=self.context, object=object_, edata=edata + ) + data = event.to_json() + print("Audit event - ", json.dumps(data, indent=4)) + + if self.producer: + self.producer.send(self.topic, data) diff --git a/command-service/src/stubs/snippets.py b/command-service/src/stubs/snippets.py new file mode 100644 index 00000000..89ffbbbd --- /dev/null +++ b/command-service/src/stubs/snippets.py @@ -0,0 +1,13 @@ +# Suspending a job in kubernetes +# def _suspend_job(self, job_name, namespace, results): +# result_status = False +# patch_cmd = """kubectl patch job/{job_name} --type=strategic --patch '{{"spec":{{"suspend":true}}}}' -n {namespace}""" +# patch_cmd = patch_cmd.format(job_name=job_name, namespace=namespace) +# patch_cmd_result = subprocess.run(patch_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,) +# if patch_cmd_result.returncode == 0: +# print(f"Job {job_name} suspension succeeded...") +# result_status = True +# else: +# print(f"Error suspending job {job_name}: {patch_cmd_result.stderr.decode()}") +# results.append(result_status) +# return results From c8faf88beab14c7a7ddd7725a08d8ccc6706fba1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 5 Sep 2024 15:33:39 +0530 Subject: [PATCH 114/311] Resolved test case issue --- api-service/src/models/DatasetDraft.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api-service/src/models/DatasetDraft.ts b/api-service/src/models/DatasetDraft.ts index 458529b6..224e3bc7 100644 --- a/api-service/src/models/DatasetDraft.ts +++ b/api-service/src/models/DatasetDraft.ts @@ -106,10 +106,6 @@ export const DatasetDraft = sequelize.define("datasets_draft", { type: DataTypes.STRING, allowNull: false }, - api_version: { - type: DataTypes.STRING, - defaultValue: "v2" - } }, { timestamps: true, createdAt: "created_date", From a47593b8dde1f59973644c96dbfc24521b17d683 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 5 Sep 2024 19:26:20 +0530 Subject: [PATCH 115/311] Merge conflicts changes --- .../connections/commandServiceConnection.ts | 4 + .../ConnectorRegisterController.ts | 2 +- .../GenerateSignedURL/GenerateSignedURL.ts | 60 +++- .../controllers/GenerateSignedURL/helper.ts | 64 ++++ .../DatasetCreate/DatasetCreate.spec.ts | 26 -- .../DatasetHealth/DatasetHealth.ts | 125 ------- .../DatasetHealthValidationSchema.json | 53 --- .../DatasetStatusTransition.ts | 265 -------------- .../ReadyToPublishSchema.json | 327 ------------------ .../RequestValidationSchema.json | 52 --- .../DatasetDelete.spec.ts | 87 ----- .../DatasetLive.spec.ts | 131 ------- .../DatasetReadyToPublish.spec.ts | 122 ------- .../DatasetRetire.spec.ts | 317 ----------------- .../DatasetStatusTransition.spec.ts | 61 ---- .../DatasetStatusTransition/Fixtures.ts | 215 ------------ 16 files changed, 125 insertions(+), 1786 deletions(-) rename api-service/src/{v2 => }/controllers/ConnectorRegister/ConnectorRegisterController.ts (100%) create mode 100644 api-service/src/controllers/GenerateSignedURL/helper.ts delete mode 100644 api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts delete mode 100644 api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json delete mode 100644 api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts delete mode 100644 api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json delete mode 100644 api-service/src/v2/controllers/DatasetStatusTransition/RequestValidationSchema.json delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts delete mode 100644 api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index 94597474..fc77fc77 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -18,4 +18,8 @@ export const executeCommand = async (id: string, command: string) => { } } return commandHttpService.post(commandPath, payload) +} + +export const registerConnector = async (requestBody: any) => { + return commandHttpService.post("/connector/v1/register", requestBody) } \ No newline at end of file diff --git a/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts similarity index 100% rename from api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts rename to api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts index 3a81f771..4844ac7e 100644 --- a/api-service/src/v2/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -6,8 +6,8 @@ import axios from "axios"; import httpStatus from "http-status"; import busboy from "busboy"; import { PassThrough } from "stream"; -import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; import { registerConnector } from "../../connections/commandServiceConnection"; +import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; export const apiId = "api.connector.register"; export const code = "FAILED_TO_REGISTER_CONNECTOR"; diff --git a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts index 6454b1fa..6182f9b3 100644 --- a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -6,20 +6,20 @@ import logger from "../../logger"; import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import GenerateURL from "./GenerateSignedURLValidationSchema.json" +import { cloudProvider } from "../../services/CloudServices"; import { config } from "../../configs/Config"; import { URLAccess } from "../../types/SampleURLModel"; -import { generatePreSignedUrl } from "./helper"; +import { v4 as uuidv4 } from "uuid"; +import path from "path"; export const apiId = "api.files.generate-url" export const code = "FILES_GENERATE_URL_FAILURE" const maxFiles = config.presigned_url_configs.maxFiles -let containerType: string; const generateSignedURL = async (req: Request, res: Response) => { const requestBody = req.body const msgid = _.get(req, ["body", "params", "msgid"]); const resmsgid = _.get(res, "resmsgid"); - containerType = _.get(req, ["body", "request", "type"]); try { const isRequestValid: Record = schemaValidation(req.body, GenerateURL) if (!isRequestValid.isValid) { @@ -46,7 +46,21 @@ const generateSignedURL = async (req: Request, res: Response) => { errCode: "BAD_REQUEST" } as ErrorObject, req, res); } - const signedUrlList = await generatePreSignedUrl(access, files, containerType) + + const { filesList, updatedFileNames } = transformFileNames(files, access) + logger.info(`Updated file names with path:${updatedFileNames}`) + + const urlExpiry: number = getURLExpiry(access) + const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)) + const signedUrlList = _.map(preSignedUrls, list => { + const fileNameWithUid = _.keys(list)[0] + return { + filePath: getFilePath(fileNameWithUid), + fileName: filesList.get(fileNameWithUid), + preSignedUrl: _.values(list)[0] + } + }) + logger.info({ apiId, requestBody, msgid, resmsgid, response: signedUrlList, message: `Sample urls generated successfully for files:${files}` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: signedUrlList }) } catch (error: any) { @@ -60,6 +74,44 @@ const generateSignedURL = async (req: Request, res: Response) => { } } +const getFilePath = (file: string) => { + return `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}` +} + +const transformFileNames = (fileList: Array, access: string): Record => { + if (access === URLAccess.Read) { + return transformReadFiles(fileList) + } + return transformWriteFiles(fileList) +} + +const transformReadFiles = (fileNames: Array) => { + const fileMap = new Map(); + const updatedFileNames = _.map(fileNames, file => { + fileMap.set(file, file) + return getFilePath(file) + }) + return { filesList: fileMap, updatedFileNames } +} + +const transformWriteFiles = (fileNames: Array) => { + const fileMap = new Map(); + const updatedFileNames = _.map(fileNames, file => { + const uuid = uuidv4().replace(/-/g, "").slice(0, 6); + const ext = path.extname(file) + const baseName = path.basename(file, ext) + const updatedFileName = `${baseName}_${uuid}${ext}` + fileMap.set(updatedFileName, file) + return getFilePath(updatedFileName) + }) + return { filesList: fileMap, updatedFileNames } + +} + +const getURLExpiry = (access: string) => { + return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry +} + const checkLimitExceed = (files: Array): boolean => { return _.size(files) > maxFiles } diff --git a/api-service/src/controllers/GenerateSignedURL/helper.ts b/api-service/src/controllers/GenerateSignedURL/helper.ts new file mode 100644 index 00000000..b6601f2a --- /dev/null +++ b/api-service/src/controllers/GenerateSignedURL/helper.ts @@ -0,0 +1,64 @@ +import { config } from "../../configs/Config"; +import * as _ from "lodash"; +import { cloudProvider } from "../../services/CloudServices"; +import { URLAccess } from "../../types/SampleURLModel"; +import path from "path"; +import { v4 as uuidv4 } from "uuid"; + +export const generatePreSignedUrl = async (access: string, files: any, containerType: string) => { + const { filesList, updatedFileNames } = transformFileNames(files, access, containerType); + const urlExpiry: number = getURLExpiry(access); + const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)); + const signedUrlList = _.map(preSignedUrls, list => { + const fileNameWithUid = _.keys(list)[0]; + return { + filePath: getFilePath(fileNameWithUid, containerType), + fileName: filesList.get(fileNameWithUid), + preSignedUrl: _.values(list)[0] + }; + }); + return signedUrlList; +} + +const getFilePath = (file: string, containerType: string) => { + const datasetUploadPath = `${config.presigned_url_configs.service}/user_uploads/${file}`; + const connectorUploadPath = `${config.cloud_config.container_prefix}/${file}`; + const paths: Record = { + "dataset": datasetUploadPath, + "connector": connectorUploadPath + }; + return paths[containerType] || datasetUploadPath; +} + +const transformFileNames = (fileList: Array, access: string, containerType: string): Record => { + if (access === URLAccess.Read) { + return transformReadFiles(fileList, containerType); + } + return transformWriteFiles(fileList, containerType); +}; + +const transformReadFiles = (fileNames: Array, containerType: string) => { + const fileMap = new Map(); + const updatedFileNames = fileNames.map(file => { + fileMap.set(file, file); + return getFilePath(file, containerType); + }); + return { filesList: fileMap, updatedFileNames }; +}; + +const transformWriteFiles = (fileNames: Array, containerType: string) => { + const fileMap = new Map(); + const updatedFileNames = fileNames.map(file => { + const uuid = uuidv4().replace(/-/g, "").slice(0, 6); + const ext = path.extname(file); + const baseName = path.basename(file, ext); + const updatedFileName = `${baseName}_${uuid}${ext}`; + fileMap.set(updatedFileName, file); + return getFilePath(updatedFileName, containerType); + }); + return { filesList: fileMap, updatedFileNames }; +}; + +const getURLExpiry = (access: string) => { + return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry; +} \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index abba6a6c..fd1aabdd 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -114,30 +114,4 @@ describe("DATASET CREATE API", () => { done(); }); }); - - - it("Failure: Master dataset not found as denorm", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(null) - }) - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ "dataset_id": "trip-data-master", "dataset_config": { "redis_db": 15 } }]) - }) - - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.VALID_DATASET) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm Master dataset not found") - res.body.error.code.should.be.eq("DATASET_DENORM_NOT_FOUND") - done(); - }); - }); - }) \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts b/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts deleted file mode 100644 index ad09e84c..00000000 --- a/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Request, Response } from "express"; -import _ from "lodash"; -import { schemaValidation } from "../../services/ValidationService"; -import DatasetHealthRequestSchema from "./DatasetHealthValidationSchema.json" -import { ErrorObject } from "../../types/ResponseModel"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { DatasetStatus, DatasetType, HealthStatus } from "../../types/DatasetModels"; -import { Dataset } from "../../models/Dataset"; -import logger from "../../logger"; -import { getInfraHealth, getProcessingHealth, getQueryHealth } from "../../services/HealthService"; -import { Datasource } from "../../models/Datasource"; - -export const apiId = "api.dataset.health"; -export const errorCode = "DATASET_HEALTH_FAILURE" - - - -const datasetHealth = async (req: Request, res: Response) => { - const resmsgid = _.get(res, "resmsgid"); - const requestBody = req.body; - const msgid = _.get(req, ["body", "params", "msgid"]); - try { - const isRequestValid: Record = schemaValidation(req.body, DatasetHealthRequestSchema) - if (!isRequestValid.isValid) { - const code = "DATASET_HEALTH_INPUT_INVALID" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - if (requestBody?.request?.dataset === "*") { - const {components, status} = await getInfraHealth(false) - return ResponseHandler.successResponse(req, res, { - status: 200, data: { - "status": status, - "details": [ - { - "category": "infra", - "status": status, - "components": components - }] - } - }); - } - - const dataset = await getLiveDatasets(requestBody?.request?.dataset) - if(_.isEmpty(dataset)) { - const code = "DATASET_HEALTH_NO_DATASETS" - const message = `There are no live dataset exists with given dataset_id: ${requestBody?.request?.dataset}` - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: message }) - return ResponseHandler.errorResponse({ - code, - message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - logger.debug(apiId, msgid, resmsgid, "dataset", dataset) - - const categories = requestBody?.request?.categories - const details = [] - if(requestBody?.request?.categories.includes("infra")) { - const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.MasterDataset; - logger.debug({isMasterDataset}) - const {components, status} = await getInfraHealth(isMasterDataset) - details.push({ - "category": "infra", - "status": status, - "components": components - }) - } - logger.debug({categories}) - if(categories.includes("processing")) { - const {components, status} = await getProcessingHealth(dataset[0]) - details.push({ - "category": "processing", - "status": status, - "components": components - }) - } - - if(categories.includes("query")) { - const datasources = await getDataSources(requestBody?.request?.dataset) - const {components, status} = await getQueryHealth(datasources, dataset[0]) - details.push({ - "category": "query", - "status": status, - "components": components - }) - } - - const allStatus = _.includes(_.map(details, (detail) => detail?.status), HealthStatus.UnHealthy) ? HealthStatus.UnHealthy: HealthStatus.Healthy - - return ResponseHandler.successResponse(req, res, { - status: 200, data: { - "status": allStatus, - "details": details - } - }); - - } catch (error: any) { - logger.error({ ...error, apiId, code: errorCode, msgid, requestBody, resmsgid }); - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: errorCode, message: "Failed to check dataset health" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } - - - -} -const getLiveDatasets = async (ids: Record): Promise> => { - return Dataset.findAll({ attributes: ["dataset_id", "status", "type"], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); -} - -const getDataSources = async (ids: Record): Promise> => { - return Datasource.findAll({ attributes: ["dataset_id", "datasource"], where: { dataset_id: ids }, raw: true }); -} - -export default datasetHealth; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json b/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json deleted file mode 100644 index 5a788259..00000000 --- a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "ts": { - "type": "string" - }, - "params": { - "type": "object", - "properties": { - "msgid": { - "type": "string" - } - }, - "required": [ - "msgid" - ], - "additionalProperties": false - }, - "request": { - "type": "object", - "properties": { - "dataset": { - "type": "string", - "minLength": 1 - }, - "categories": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "infra", - "processing", - "query" - ], - "required": [ - "category" - ] - } - } - }, - "required": [ - "categories" - ] - } - } -} \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts deleted file mode 100644 index b902006a..00000000 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { Request, Response } from "express"; -import _ from "lodash"; -import logger from "../../logger"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { generateDataSource, getDataset, getDraftDataset, setReqDatasetId } from "../../services/DatasetService"; -import { ErrorObject } from "../../types/ResponseModel"; -import { schemaValidation } from "../../services/ValidationService"; -import StatusTransitionSchema from "./RequestValidationSchema.json"; -import ReadyToPublishSchema from "./ReadyToPublishSchema.json" -import httpStatus from "http-status"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { DatasetSourceConfigDraft } from "../../models/DatasetSourceConfigDraft"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import { Dataset } from "../../models/Dataset"; -import { DatasetAction, DatasetStatus, DatasetType } from "../../types/DatasetModels"; -import { DatasetSourceConfig } from "../../models/DatasetSourceConfig"; -import { Datasource } from "../../models/Datasource"; -import { DatasetTransformations } from "../../models/Transformation"; -import { executeCommand } from "../../connections/commandServiceConnection"; -import { druidHttpService } from "../../connections/druidConnection"; -import { sequelize } from "../../connections/databaseConnection"; - -export const apiId = "api.datasets.status-transition"; -export const errorCode = "DATASET_STATUS_TRANSITION_FAILURE" - -const allowedTransitions = { - Delete: [DatasetStatus.Draft, DatasetStatus.ReadyToPublish], - ReadyToPublish: [DatasetStatus.Draft], - Live: [DatasetStatus.ReadyToPublish], - Retire: [DatasetStatus.Live], -} - -const statusTransitionCommands = { - Delete: ["DELETE_DRAFT_DATASETS"], - ReadyToPublish: ["VALIDATE_DATASET_CONFIGS"], - Live: ["GENERATE_INGESTION_SPEC", "PUBLISH_DATASET"], - Retire: ["CHECK_DATASET_IS_DENORM", "SET_DATASET_TO_RETIRE", "DELETE_SUPERVISORS", "RESTART_PIPELINE"] -} - -const datasetStatusTransition = async (req: Request, res: Response) => { - const requestBody = req.body - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - let transact; - try { - const { dataset_id, status } = _.get(requestBody, "request"); - setReqDatasetId(req, dataset_id) - - const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) - if (!isRequestValid.isValid) { - const code = "DATASET_STATUS_TRANSITION_INVALID_INPUT" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetRecord = await fetchDataset({ status, dataset_id }) - if (_.isEmpty(datasetRecord)) { - const code = "DATASET_NOT_FOUND" - const errorMessage = getErrorMessage(status, code) - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `${errorMessage} for dataset:${dataset_id}` }) - return ResponseHandler.errorResponse({ - code, - message: errorMessage, - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - } - - const allowedStatus = _.get(allowedTransitions, status) - const datasetStatus = _.get(datasetRecord, "status") - if (!_.includes(allowedStatus, datasetStatus)) { - const code = `DATASET_${_.toUpper(status)}_FAILURE` - const errorMessage = getErrorMessage(status, "STATUS_INVALID") - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `${errorMessage} for dataset:${dataset_id} status:${datasetStatus} with status transition to ${status}` }) - return ResponseHandler.errorResponse({ - code, - message: errorMessage, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const transitionCommands = _.get(statusTransitionCommands, status) - transact = await sequelize.transaction() - await executeTransition({ transitionCommands, dataset: datasetRecord, transact }) - - await transact.commit(); - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Dataset status transition to ${status} successful with id:${dataset_id}` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); - } catch (error: any) { - transact && await transact.rollback(); - const code = _.get(error, "code") || errorCode - logger.error(error, apiId, msgid, code, requestBody, resmsgid) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to perform status transition on datasets" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const fetchDataset = async (configs: Record) => { - const { dataset_id, status } = configs - if (_.includes([DatasetAction.ReadyToPublish, DatasetAction.Delete], status)) { - return getDraftDatasetRecord(dataset_id) - } - if (_.includes([DatasetAction.Live], status)) { - return getDraftDataset(dataset_id) - } - if (_.includes([DatasetAction.Retire], status)) { - return getDataset(dataset_id) - } -} - -const executeTransition = async (configs: Record) => { - const { transitionCommands, dataset, transact } = configs - for (const command of transitionCommands) { - const commandWorkflow = _.get(commandExecutors, command); - await commandWorkflow({ dataset, transact }); - } -} - -//VALIDATE_DATASET_CONFIGS -const validateDataset = async (configs: Record) => { - const { dataset } = configs - const datasetValid: Record = schemaValidation(dataset, ReadyToPublishSchema) - if (!datasetValid.isValid) { - throw { - code: "DATASET_CONFIGS_INVALID", - message: datasetValid.message, - errCode: "BAD_REQUEST", - statusCode: 400 - } - } - await DatasetDraft.update({ status: DatasetStatus.ReadyToPublish }, { where: { id: dataset.id } }) -} - -//DELETE_DRAFT_DATASETS -const deleteDataset = async (configs: Record) => { - const { dataset, transact } = configs - const { id } = dataset - await deleteDraftRecords({ dataset_id: id, transact }) -} - -const deleteDraftRecords = async (config: Record) => { - const { dataset_id, transact } = config; - await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction: transact }) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id }, transaction: transact }) - await DatasourceDraft.destroy({ where: { dataset_id }, transaction: transact }) - await DatasetDraft.destroy({ where: { id: dataset_id }, transaction: transact }) -} - -//GENERATE_INGESTION_SPEC -const generateIngestionSpec = async (config: Record) => { - const { dataset } = config; - const dataSource = await generateDataSource(dataset); - return DatasourceDraft.upsert(dataSource) -} - -//PUBLISH_DATASET -const publishDataset = async (configs: Record) => { - const { dataset } = configs - const { dataset_id } = dataset - await executeCommand(dataset_id, "PUBLISH_DATASET"); -} - -//CHECK_DATASET_IS_DENORM -const checkDatasetDenorm = async (payload: Record) => { - const { dataset } = payload - const { dataset_id, type } = dataset - if (type === DatasetType.MasterDataset) { - const liveDatasets = await Dataset.findAll({ attributes: ["denorm_config"], raw: true }) || [] - const draftDatasets = await DatasetDraft.findAll({ attributes: ["denorm_config"], raw: true }) || [] - _.forEach([...liveDatasets, ...draftDatasets], datasets => { - _.forEach(_.get(datasets, "denorm_config.denorm_fields"), denorms => { - if (_.get(denorms, "dataset_id") === dataset_id) { - logger.error(`Failed to retire dataset as it is used by other datasets:${dataset_id}`) - throw { - code: "DATASET_IN_USE", - errCode: "BAD_REQUEST", - message: "Failed to retire dataset as it is used by other datasets", - statusCode: 400 - } - } - }) - }) - } -} - -//SET_DATASET_TO_RETIRE -const setDatasetRetired = async (config: Record) => { - const { dataset, transact } = config; - const { dataset_id } = dataset - await Dataset.update({ status: DatasetStatus.Retired }, { where: { dataset_id }, transaction: transact }) - await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id }, transaction: transact }) - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id }, transaction: transact }) - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id }, transaction: transact }) -} - -//DELETE_SUPERVISORS -const deleteSupervisors = async (configs: Record) => { - const { dataset } = configs - const { type, dataset_id } = dataset - try { - if (type !== DatasetType.MasterDataset) { - const datasourceRefs = await Datasource.findAll({ where: { dataset_id }, attributes: ["datasource_ref"], raw: true }) - for (const sourceRefs of datasourceRefs) { - const datasourceRef = _.get(sourceRefs, "datasource_ref") - await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceRef}/terminate`) - logger.info(`Datasource ref ${datasourceRef} deleted from druid`) - } - } - } catch (error: any) { - logger.error({ error: _.get(error, "message"), message: `Failed to delete supervisors for dataset:${dataset_id}` }) - } -} - -//RESTART_PIPELINE -const restartPipeline = async (config: Record) => { - const dataset_id = _.get(config, ["dataset", "dataset_id"]) - return executeCommand(dataset_id, "RESTART_PIPELINE") -} - -const commandExecutors = { - DELETE_DRAFT_DATASETS: deleteDataset, - PUBLISH_DATASET: publishDataset, - GENERATE_INGESTION_SPEC: generateIngestionSpec, - CHECK_DATASET_IS_DENORM: checkDatasetDenorm, - SET_DATASET_TO_RETIRE: setDatasetRetired, - DELETE_SUPERVISORS: deleteSupervisors, - RESTART_PIPELINE: restartPipeline, - VALIDATE_DATASET_CONFIGS: validateDataset -} - -const getDraftDatasetRecord = async (dataset_id: string) => { - return DatasetDraft.findOne({ where: { id: dataset_id }, raw: true }); -} - -const errorMessage = { - DATASET_NOT_FOUND: { - Delete: "Dataset not found to delete", - Retire: "Dataset not found to retire", - ReadyToPublish: "Dataset not found to perform status transition to ready to publish", - Live: "Dataset not found to perform status transition to live" - }, - STATUS_INVALID: { - Delete: "Failed to Delete dataset", - Retire: "Failed to Retire dataset as it is not in live state", - ReadyToPublish: "Failed to mark dataset Ready to publish as it not in draft state", - Live: "Failed to mark dataset Live as it is not in ready to publish state" - } -} - -const getErrorMessage = (status: string, code: string) => { - return _.get(errorMessage, [code, status]) || "Failed to perform status transition" -} - -export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json deleted file mode 100644 index 8ffd9591..00000000 --- a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ /dev/null @@ -1,327 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "dataset_id": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "integer" - }, - "status": { - "type": "string", - "enum": ["Draft"] - }, - "type": { - "type": "string", - "enum": ["dataset", "master-dataset"] - }, - "name": { - "type": "string", - "minLength": 1 - }, - "version_key": { - "type": "string", - "minLength": 1 - }, - "validation_config": { - "type": "object", - "properties": { - "validate": { - "type": "boolean" - }, - "mode": { - "type": "string", - "enum": ["Strict", "IgnoreNewFields"] - } - }, - "required": ["validate", "mode"] - }, - "extraction_config": { - "type": "object", - "properties": { - "is_batch_event": { - "type": "boolean" - }, - "extraction_key": { - "type": "string" - }, - "batch_id": { - "type": "string" - }, - "dedup_config": { - "type": "object", - "properties": { - "drop_duplicates": { - "type": "boolean" - }, - "dedup_key": { - "type": "string" - }, - "dedup_period": { - "type": "integer" - } - }, - "if": { - "properties": { - "drop_duplicates": { - "const": true - } - } - }, - "then": { - "properties": { - "dedup_key": { - "minLength": 1 - } - } - }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], - "additionalProperties": false - } - }, - "if": { - "properties": { - "is_batch_event": { - "const": true - } - } - }, - "then": { - "properties": { - "extraction_key": { - "minLength": 1 - }, - "batch_id": { - "minLength": 1 - } - } - }, - "required": [ - "is_batch_event", - "extraction_key", - "dedup_config" - ], - "additionalProperties": false - }, - "dedup_config": { - "type": "object", - "properties": { - "drop_duplicates": { - "type": "boolean" - }, - "dedup_key": { - "type": "string" - }, - "dedup_period": { - "type": "integer" - } - }, - "if": { - "properties": { - "drop_duplicates": { - "const": true - } - } - }, - "then": { - "properties": { - "dedup_key": { - "minLength": 1 - } - } - }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], - "additionalProperties": false - }, - "data_schema": { - "type": "object", - "properties": { - "$schema": { - "type": "string", - "minLength": 1 - }, - "type": { - "type": "string", - "minLength": 1 - }, - "properties": { - "type": "object", - "minProperties": 1 - }, - "required": { - "type": "array" - }, - "additionalProperties": { - "type": "boolean" - } - }, - "required": ["type", "properties"], - "additionalProperties": false - }, - "denorm_config": { - "type": "object", - "properties": { - "redis_db_host": { - "type": "string", - "minLength": 1 - }, - "redis_db_port": { - "type": "integer" - }, - "denorm_fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "denorm_key": { - "type": "string", - "minLength": 1 - }, - "denorm_out_field": { - "type": "string", - "minLength": 1 - }, - "dataset_name": { - "type": "string", - "minLength": 1 - }, - "dataset_id": { - "type": "string", - "minLength": 1 - }, - "redis_db": { - "type": "integer" - } - }, - "required": [ - "denorm_key", - "denorm_out_field", - "dataset_id" - ], - "additionalProperties": false - } - } - }, - "required": ["redis_db_host", "redis_db_port", "denorm_fields"], - "additionalProperties": false - }, - "router_config": { - "type": "object", - "properties": { - "topic": { - "type": "string", - "minLength": 1 - } - }, - "required": ["topic"], - "additionalProperties": false - }, - "dataset_config": { - "type": "object", - "properties": { - "mergedEvent": { - "type": "object" - }, - "configurations": { - "type": "object" - }, - "dataMappings": { - "type": "object" - }, - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string", - "minLength": 1 - }, - "entry_topic": { - "type": "string", - "minLength": 1 - }, - "redis_db_host": { - "type": "string", - "minLength": 1 - }, - "exclude_fields": { - "type": "array" - }, - "redis_db_port": { - "type": "integer" - }, - "index_data": { - "type": "boolean" - }, - "redis_db": { - "type": "integer" - }, - "files_upload_path": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - }, - "required": [ - "timestamp_key", - "entry_topic", - "redis_db_host", - "redis_db_port", - "redis_db", - "index_data" - ], - "additionalProperties": false - }, - "tags": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, - "client_state": { - "type": "object" - } - }, - "if": { - "properties": { - "type": { - "const": "master-dataset" - } - } - }, - "then": { - "properties": { - "dataset_config": { - "properties": { - "data_key": { - "minLength": 1 - } - }, - "required": ["data_key"] - } - } - }, - "required": [ - "dataset_id", - "type", - "name", - "data_schema", - "id", - "validation_config", - "extraction_config", - "dataset_config", - "denorm_config", - "dedup_config", - "router_config", - "client_state", - "version_key", - "status" - ] -} diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/RequestValidationSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/RequestValidationSchema.json deleted file mode 100644 index c7455af6..00000000 --- a/api-service/src/v2/controllers/DatasetStatusTransition/RequestValidationSchema.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string", - "enum": [ - "api.datasets.status-transition" - ] - }, - "ver": { - "type": "string" - }, - "ts": { - "type": "string" - }, - "params": { - "type": "object", - "properties": { - "msgid": { - "type": "string" - } - }, - "required": [ - "msgid" - ], - "additionalProperties": false - }, - "request": { - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "minLength": 1 - }, - "status": { - "type": "string", - "enum": [ - "ReadyToPublish", - "Retire", - "Delete", - "Live" - ] - } - }, - "additionalProperties": false, - "required": [ - "dataset_id", - "status" - ] - } - } - } \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts deleted file mode 100644 index 566f5316..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts +++ /dev/null @@ -1,87 +0,0 @@ -import app from "../../../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; -import { TestInputsForDatasetStatusTransition } from "./Fixtures"; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; -import { DatasourceDraft } from "../../../models/DatasourceDraft"; -import { sequelize } from "../../../connections/databaseConnection"; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" - -describe("DATASET STATUS TRANSITION DELETE", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset status transition success: When the action is to Delete draft datasets", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Draft", id: "telemetry.1" }) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfigDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "destroy", () => { - return Promise.resolve({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.message.should.be.eq("Dataset status transition to Delete successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") - done(); - }); - }); - - it("Dataset status transition failure: When dataset is not found to delete", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve() - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to delete") - res.body.error.code.should.be.eq("DATASET_NOT_FOUND") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts deleted file mode 100644 index 7c6122ba..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ /dev/null @@ -1,131 +0,0 @@ -import app from "../../../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; -import { TestInputsForDatasetStatusTransition } from "./Fixtures"; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import { commandHttpService } from "../../../connections/commandServiceConnection"; -import { sequelize } from "../../../connections/databaseConnection"; -import { DatasourceDraft } from "../../../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" - -describe("DATASET STATUS TRANSITION LIVE", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset status transition success: When the action is to set dataset live", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_READ) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) - chai.spy.on(DatasourceDraft, "upsert", () => { - return Promise.resolve({}) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.resolve({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.message.should.be.eq("Dataset status transition to Live successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") - done(); - }); - }); - - it("Dataset status transition failure: When dataset is not found to publish", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve() - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to perform status transition to live") - res.body.error.code.should.be.eq("DATASET_NOT_FOUND") - done(); - }) - }) - - it("Dataset status transition failure: When the command api call to publish dataset fails", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "ReadyToPublish" }) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.reject() - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); - - it("Dataset status transition failure: When the dataset to publish is in draft state", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Draft" }) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq("DATASET_LIVE_FAILURE") - res.body.error.message.should.be.eq("Failed to mark dataset Live as it is not in ready to publish state") - done(); - }); - }); - -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts deleted file mode 100644 index 2a1ff7a3..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ /dev/null @@ -1,122 +0,0 @@ -import app from "../../../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; -import { TestInputsForDatasetStatusTransition } from "./Fixtures"; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import { sequelize } from "../../../connections/databaseConnection"; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" - -describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset status transition success: When the action is make dataset ready to publish", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_READY_TO_PUBLISH) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") - done(); - }); - }); - - it("Dataset status transition failure: When dataset is not found to ready to publish", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve() - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to perform status transition to ready to publish") - res.body.error.code.should.be.eq("DATASET_NOT_FOUND") - done(); - }); - }); - - it("Dataset status transition failure: When dataset is already ready to publish", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({dataset_id:"telemetry", status:"ReadyToPublish"}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to mark dataset Ready to publish as it not in draft state") - res.body.error.code.should.be.eq("DATASET_READYTOPUBLISH_FAILURE") - done(); - }); - }); - - - it("Dataset status transition failure: Configs invalid to set status to ready to publish", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(TestInputsForDatasetStatusTransition.INVALID_SCHEMA_FOR_READY_TO_PUBLISH) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - expect(res.body.error.message).to.match(/^#required must have(.+)/) - res.body.error.code.should.be.eq("DATASET_CONFIGS_INVALID") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts deleted file mode 100644 index d231b91c..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ /dev/null @@ -1,317 +0,0 @@ -import app from "../../../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; -import { TestInputsForDatasetStatusTransition } from "./Fixtures"; -import { Dataset } from "../../../models/Dataset"; -import { DatasetDraft } from "../../../models/DatasetDraft"; -import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; -import { Datasource } from "../../../models/Datasource"; -import { commandHttpService } from "../../../connections/commandServiceConnection"; -import { druidHttpService } from "../../../connections/druidConnection"; -import { sequelize } from "../../../connections/databaseConnection"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" - -describe("DATASET STATUS TRANSITION RETIRE", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset status transition success: When the action is to Retire dataset", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) - }) - chai.spy.on(DatasetTransformations, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfig, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) - }) - chai.spy.on(druidHttpService, "post", () => { - return Promise.resolve({}) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.resolve({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.message.should.be.eq("Dataset status transition to Retire successful") - res.body.result.dataset_id.should.be.eq("telemetry") - done(); - }); - }); - - it("Dataset status transition success: When the action is to Retire master dataset", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "master-dataset" }) - }) - chai.spy.on(DatasetTransformations, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfig, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.resolve({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.message.should.be.eq("Dataset status transition to Retire successful") - res.body.result.dataset_id.should.be.eq("telemetry") - done(); - }); - }); - - it("Dataset status transition successs: Dataset successfully retired on delete supervisors failure", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) - }) - chai.spy.on(DatasetTransformations, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfig, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) - }) - chai.spy.on(druidHttpService, "post", () => { - return Promise.reject({}) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.resolve({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.message.should.be.eq("Dataset status transition to Retire successful") - res.body.result.dataset_id.should.be.eq("telemetry") - done(); - }); - }); - - it("Dataset status transition failure: When dataset is not found to retire", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve() - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to retire") - res.body.error.code.should.be.eq("DATASET_NOT_FOUND") - done(); - }) - }) - - it("Dataset status transition failure: When dataset is already retired", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Retired", type: "dataset" }) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to Retire dataset as it is not in live state") - res.body.error.code.should.be.eq("DATASET_RETIRE_FAILURE") - done(); - }) - }) - - it("Dataset status transition failure: When dataset to retire is used by other datasets", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", type: "master-dataset", status: "Live" }) - }) - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "telemetry" }] } }]) - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "telemetry" }] } }]) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to retire dataset as it is used by other datasets") - res.body.error.code.should.be.eq("DATASET_IN_USE") - done(); - }); - }); - - it("Dataset status transition failure: When setting retire status to live records fail", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); - - it("Dataset status transition failure: Failed to restart pipeline", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", type: "dataset", status: "Live", }) - }) - chai.spy.on(DatasetTransformations, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfig, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts deleted file mode 100644 index ee589998..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts +++ /dev/null @@ -1,61 +0,0 @@ -import app from "../../../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { describe, it } from 'mocha'; -import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; -import { TestInputsForDatasetStatusTransition } from "./Fixtures"; -import { DatasetDraft } from "../../../models/DatasetDraft"; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6" - -describe("DATASET STATUS TRANSITION API", () => { - - afterEach(() => { - chai.spy.restore(); - }); - - it("Dataset status transition failure: Invalid request payload provided", (done) => { - - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - expect(res.body.error.message).to.match(/^#properties\/request(.+)$/) - res.body.error.code.should.be.eq("DATASET_STATUS_TRANSITION_INVALID_INPUT") - done(); - }); - }); - - it("Dataset status transition failure: Connection to the database failed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.reject() - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); -}) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts deleted file mode 100644 index ee8e9dee..00000000 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ /dev/null @@ -1,215 +0,0 @@ -export const TestInputsForDatasetStatusTransition = { - VALID_SCHEMA_FOR_DELETE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Delete" - } - }, - VALID_SCHEMA_FOR_LIVE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Live" - } - }, - VALID_SCHEMA_FOR_RETIRE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry", - "status": "Retire" - } - }, - INVALID_SCHEMA: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "" - } - }, - VALID_REQUEST_FOR_READY_FOR_PUBLISH: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "ReadyToPublish" - } - }, - VALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "dataset", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "batch_id": "id", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 3783 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 3783 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "ets": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - }, - INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - }, - - VALID_SCHEMA_FOR_LIVE_READ: { - "dataset_id": "sb-ddd", - "type": "dataset", - "name": "sb-telemetry2", - "status": "ReadyToPublish", - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "ets": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - }, - "tags": [] - } -} \ No newline at end of file From 39b14e317d2ddaeb9d6598399c5e0f58f168b8bc Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:01:47 +0530 Subject: [PATCH 116/311] #OBS-I165: added userInfo from token to request object --- api-service/src/middlewares/RBAC_middleware.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index d36babf0..22bd40b3 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -152,6 +152,7 @@ export default { ); } if (decoded && _.isObject(decoded)) { + (req as any).userInfo = decoded; const action = (req as any).id; const hasAccess = decoded?.roles?.some( (role: string) => From b85cabfc4442186898b158af9b15b31a9d481191 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:03:41 +0530 Subject: [PATCH 117/311] #OBS-I165: updated telemetry to use user role --- api-service/src/services/telemetry.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index d4eca61b..f24995a9 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -10,15 +10,15 @@ const telemetryTopic = _.get(appConfig, "telemetry_dataset"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const getDefaults = () => { +const getDefaults = (userInfo:any) => { return { eid: "AUDIT", ets: Date.now(), ver: "1.0.0", mid: v4(), actor: { - id: "SYSTEM", - type: "User" + id: userInfo?.id || "SYSTEM", + type: userInfo?.roles[0] || "User", }, context: { env, @@ -128,7 +128,7 @@ export const processAuditEvents = (request: Request) => { _.set(auditEvent, "edata.transition.toState", toState); _.set(auditEvent, "edata.transition.fromState", fromState); } - const telemetryEvent = getDefaults(); + const telemetryEvent = getDefaults((request as any)?.userInfo); _.set(telemetryEvent, "edata", edata); _.set(telemetryEvent, "object", { ...(object.id && object.type && { ...object, ver: "1.0.0" }) }); sendTelemetryEvents(telemetryEvent); From 627a60bfed3554ca6b5478b7a4f9a6b98b3cfd9d Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:08:18 +0530 Subject: [PATCH 118/311] #OBS-I165: updated datasetCreate api to add userRole as created_by --- api-service/src/controllers/DatasetCreate/DatasetCreate.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 2867457b..2e476b7c 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,6 +34,8 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) + const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; + _.set(draftDataset, "created_by", userRole); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } From e33fb97f6c4493c6534598a4660527cf7f10fe74 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:12:00 +0530 Subject: [PATCH 119/311] #OBS-I165: added userRole when migrating and create table from live --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index e1853e57..5e3a869c 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,8 +30,9 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; + const userRole = (req as any)?.userInfo?.roles[0]; const attributes = !fields ? defaultFields : _.split(fields, ","); - const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userRole) : await readDataset(dataset_id, attributes) if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } @@ -41,19 +42,19 @@ const datasetRead = async (req: Request, res: Response) => { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } -const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { +const readDraftDataset = async (datasetId: string, attributes: string[], userRole: string): Promise => { const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); - const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userRole) return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if (liveDataset) { - const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userRole) return _.pick(dataset, attributes); } From 27c0940d29b40c5276afd4a5c9ab68db02764097 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:14:04 +0530 Subject: [PATCH 120/311] #OBS-I165: added userRole for data copy api --- api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 9308dd71..8ddc9fe2 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,6 +40,8 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); + const userRole = (req as any)?.userInfo?.roles[0]; + _.set(dataset, "created_by", userRole); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { From dde0858731df03451c996e42fa30d1e71e0bb963 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:15:04 +0530 Subject: [PATCH 121/311] #OBS-I165: added userRole for dataset Update --- api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 274ad3c4..679b3fcc 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,6 +59,7 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); + _.set(draftDataset, "updated_by", (req as any)?.userInfo?.roles[0] ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } From 8e946954b07d5f61c3c2a8deeb7cf882033b00a4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:15:49 +0530 Subject: [PATCH 122/311] #OBS-I165: added userRole for dataset Import --- api-service/src/controllers/DatasetImport/DatasetImport.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index e390d8bd..97314ecc 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,18 +15,21 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } + const userRole = (req as any)?.userInfo?.roles[0]; + _.set(datasetPayload, "created_by", userRole); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await importDataset(updatedDataset, overwrite); + const dataset = await importDataset(updatedDataset, overwrite, userRole); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const importDataset = async (dataset: Record, overwrite: string | any) => { +const importDataset = async (dataset: Record, overwrite: string | any, userRole : string) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { + _.set(dataset, "updated_by", userRole); const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) From 7556358af993323d5ed9d68f0e958a0be471661b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:16:03 +0530 Subject: [PATCH 123/311] #OBS-I165: added userRole for dataset status transition --- .../DatasetStatusTransition.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index c2a18021..eb03c7b5 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,6 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; validateDataset(dataset, dataset_id, status); switch (status) { @@ -62,13 +63,13 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await deleteDataset(dataset); break; case "ReadyToPublish": - await readyForPublish(dataset); + await readyForPublish(dataset, userRole); break; case "Live": - await publishDataset(dataset); + await publishDataset(dataset, userRole); break; case "Retire": - await retireDataset(dataset); + await retireDataset(dataset, userRole); break; case "Archive": await archiveDataset(dataset); @@ -90,7 +91,7 @@ const deleteDataset = async (dataset: Record) => { } -const readyForPublish = async (dataset: Record) => { +const readyForPublish = async (dataset: Record, updated_by: any) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) @@ -99,7 +100,14 @@ const readyForPublish = async (dataset: Record) => { if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } - _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { + _.set(draftDataset, "updated_by", updated_by); + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue ,key) => { + if (key === "created_by"|| key === "updated_by") { + if (objValue !== "SYSTEM") { + return objValue; + } + return srcValue; + } if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { return objValue; } @@ -126,10 +134,11 @@ const readyForPublish = async (dataset: Record) => { * * @param dataset */ -const publishDataset = async (dataset: Record) => { +const publishDataset = async (dataset: Record, userRole: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - + _.set(draftDataset, ["updated_by"], userRole); + console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) @@ -208,10 +217,10 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record) => { +const retireDataset = async (dataset: Record, updated_by: any) => { await canRetireIfMasterDataset(dataset); - await datasetService.retireDataset(dataset); + await datasetService.retireDataset(dataset, updated_by); await restartPipeline(dataset); } From 40987b3268a19254a9d6e75f4f66d6f5e762d2a4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:17:05 +0530 Subject: [PATCH 124/311] #OBS-I165: modified Dataset Service to update the userRoles --- api-service/src/services/DatasetService.ts | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 2e800212..4bb27ed7 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -92,9 +92,10 @@ class DatasetService { return responseData; } - migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { + migrateDraftDataset = async (datasetId: string, dataset: Record, userRole: string): Promise => { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); + _.set(draftDataset, "updated_by", userRole); const transaction = await sequelize.transaction(); try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); @@ -166,7 +167,7 @@ class DatasetService { } } - createDraftDatasetFromLive = async (dataset: Model) => { + createDraftDatasetFromLive = async (dataset: Model, userRole: string) => { const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config: any = _.get(dataset, "dataset_config"); @@ -232,6 +233,7 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft + draftDataset["created_by"] = userRole; const result = await DatasetDraft.create(draftDataset); return _.get(result, "dataValues") } @@ -256,14 +258,14 @@ class DatasetService { } } - retireDataset = async (dataset: Record) => { + retireDataset = async (dataset: Record, updatedBy: any) => { const transaction = await sequelize.transaction(); try { - await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); - await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await Dataset.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { id: dataset.id }, transaction }); + await DatasetSourceConfig.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); + await Datasource.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); } catch (err: any) { @@ -321,30 +323,39 @@ class DatasetService { private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.create(draftDatasource, { transaction }) } From e013c29bf13289afeef9861495b743f54d9f6243 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 18:05:17 +0530 Subject: [PATCH 125/311] #OBS-I165: added permission for queryTemplateUpdate api --- api-service/src/middlewares/RBAC_middleware.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 22bd40b3..92d352c8 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -70,6 +70,7 @@ const accessControl: AccessControl = { permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, + permissions.QueryTemplateUpdate, permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, @@ -89,6 +90,7 @@ const accessControl: AccessControl = { permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, + permissions.QueryTemplateUpdate, permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, @@ -111,6 +113,7 @@ const accessControl: AccessControl = { permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, + permissions.QueryTemplateUpdate, permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, From a88f30cdfd05e17b2d40dd0ee776c913e0971e58 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 18:10:33 +0530 Subject: [PATCH 126/311] #OBS-I165: added userRole for query template create and update --- .../controllers/CreateQueryTemplate/CreateTemplateController.ts | 1 + .../controllers/UpdateQueryTemplate/UpdateTemplateController.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 90808e7d..94b4cffd 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,6 +42,7 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); + _.set(data, "created_by", (req as any)?.userInfo?.roles[0]); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 114d370e..30f3c5c5 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,7 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - + requestBody.request.updated_by = (req as any)?.userInfo?.roles[0]; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); From c48efd77113ca6eb0a83243549ae5815c057c388 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 08:39:27 +0530 Subject: [PATCH 127/311] #OBS-I165: added userRole for alerts api --- api-service/src/controllers/Alerts/Alerts.ts | 7 ++++++- api-service/src/services/managers/index.ts | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 278ac586..131fc9a2 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,6 +13,7 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); + _.set(alertPayload, "created_by", (req as any)?.userInfo?.roles[0]); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -30,6 +31,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -87,6 +89,7 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); + _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -104,11 +107,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); if (rulePayload.status == "live") { + _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } }); + const userRole = (req as any)?.userInfo?.roles[0]; + await Alert.update({ ...updatedPayload, status: "draft", updated_by: userRole }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index 84b666d9..e6cf822f 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -24,16 +24,16 @@ const getService = (manager: string) => { }; export const publishAlert = async (payload: Record) => { - const { id, manager } = payload; + const { id, manager, updated_by } = payload; const service = getService(manager); const publishResponse = await service.publishAlert(payload) - await updateStatus(id, "live"); + await updateStatus(id, "live", updated_by); return publishResponse; }; -const updateStatus = (id: string, status: string) => { - return Alert.update({ status }, { where: { id } }); +const updateStatus = (id: string, status: string, updated_by: string) => { + return Alert.update({ status, updated_by }, { where: { id } }); } const deleteRule = (id: string) => { @@ -41,7 +41,7 @@ const deleteRule = (id: string) => { } export const deleteAlertRule = async (payload: Record, hardDelete: boolean) => { - const { id, manager, status } = payload; + const { id, manager, status, updated_by } = payload; if (status == "live") { try { @@ -56,7 +56,7 @@ export const deleteAlertRule = async (payload: Record, hardDelete: return deleteRule(id); } - return updateStatus(id, "retired"); + return updateStatus(id, "retired", updated_by); } From 5e49317f898bad1fc93d89456a35b76ce90469c6 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 08:39:46 +0530 Subject: [PATCH 128/311] #OBS-I165: added userRole for notifications api --- .../controllers/NotificationChannel/Notification.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index f6db4fd4..949c8670 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,6 +12,8 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; + const userRole = (request as any)?.userInfo?.role[0]; + _.set(payload, "created_by", userRole); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) @@ -32,6 +34,8 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } + const userRole = (request as any)?.userInfo?.role[0]; + _.set(updatedPayload, "updated_by", userRole); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -74,7 +78,8 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - await Notification.update({ status: "retired" }, { where: { id } }) + const userRole = (request as any)?.userInfo?.role[0]; + await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) @@ -91,7 +96,8 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - Notification.update({ status: "live" }, { where: { id } }); + const userRole = (request as any)?.userInfo?.role[0]; + Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) From 428db1b1ca825010a3ef7a5128a585233dfbeb57 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 08:40:04 +0530 Subject: [PATCH 129/311] #OBS-I165: added userRole for silences api --- api-service/src/controllers/Alerts/Silence.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 32e2c531..fd2bee31 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,12 +20,14 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); + const userRole = (request as any)?.userInfo?.roles[0]; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, alert_id: alertId, start_time: start_date, end_time: end_date, + created_by : userRole, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); @@ -78,10 +80,12 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); + const userRole = (request as any)?.userInfo?.roles[0]; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, - end_time: updatedEndTime + end_time: updatedEndTime, + updated_by: userRole, } const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) From d2bffb6b01061fe765365aa43e3b8ae2faefa0ff Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 11 Sep 2024 12:48:03 +0530 Subject: [PATCH 130/311] #OBS-I86 : commented routes have regex --- api-service/src/app.ts | 2 +- api-service/src/routes/DruidProxyRouter.ts | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 962331e9..27b9fce6 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -22,7 +22,7 @@ app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); app.use("/", metricRouter); -app.use("*", ResponseHandler.routeNotFound); +// app.use("*", ResponseHandler.routeNotFound); app.use(obsrvErrorHandler); app.listen(config.api_port, () => { diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index ca1aec91..70fa3ee6 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -9,15 +9,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; export const druidProxyRouter = express.Router(); // Send a 410 Gone response to all V1 API calls -druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) -// Druid Proxy APIs for Metabase integration -druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) -druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) -druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) -druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +// // Druid Proxy APIs for Metabase integration +// druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) +// druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) +// druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) +// druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) +// druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file From 23abcc40839c947b195253a920b03b1797677d85 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 11 Sep 2024 16:41:04 +0530 Subject: [PATCH 131/311] #OBS-I77 : query api changes to check datasource avialability v2 --- .../src/controllers/DataOut/QueryValidator.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index e569bdb3..9058603e 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -4,10 +4,11 @@ import * as _ from "lodash"; import moment from "moment"; import { getDatasourceList } from "../../services/DatasourceService"; import logger from "../../logger"; -import { getDatasourceListFromDruid } from "../../connections/druidConnection"; +import { druidHttpService, getDatasourceListFromDruid } from "../../connections/druidConnection"; import { apiId } from "./DataOutController"; import { ErrorObject } from "../../types/ResponseModel"; import { Parser } from "node-sql-parser"; +import { obsrvError } from "../../types/ObsrvError"; const parser = new Parser(); const momentFormat = "YYYY-MM-DD HH:MM:SS"; @@ -155,23 +156,35 @@ const validateQueryRules = (queryPayload: any, limits: any) => { : { message: "Invalid date range! the date range cannot be a null value", statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange }; }; -const getDataSourceRef = async (datasetId: string, granularity?: string) => { +const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { const dataSources = await getDatasourceList(datasetId) if (_.isEmpty(dataSources)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Datasource ${datasetId} not available in datasource live table`, code: errCode.notFound }) throw { message: `Datasource ${datasetId} not available for querying`, statusCode: 404, errCode: "NOT_FOUND", code: errCode.notFound } as ErrorObject; } - const record = dataSources.filter((record: any) => { - const aggregatedRecord = _.get(record, "dataValues.metadata.aggregated") - if (granularity) - return aggregatedRecord && _.get(record, "dataValues.metadata.granularity") === granularity; + const record = dataSources.find((record: any) => { + const metadata = _.get(record, "dataValues.metadata", {}); + const { aggregated, granularity } = metadata; + if (!aggregated) { + return true; + } + return aggregated && srcGranularity ? granularity === srcGranularity : false; }); - return record[0]?.dataValues?.datasource_ref + return _.get(record, ["dataValues", "datasource_ref"]) +} + +const checkSupervisorAvailability = async (datasourceRef: string) => { + const { data } = await druidHttpService.get('/druid/coordinator/v1/loadstatus'); + const datasourceLoad = _.get(data, datasourceRef) + if (!(datasourceLoad && datasourceLoad === 100)) { + throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not fully available to query", "RANGE_NOT_SATISFIABLE", 416) + } } const setDatasourceRef = async (datasetId: string, payload: any): Promise => { const granularity = _.get(payload, "context.aggregationLevel") const datasourceRef = await getDataSourceRef(datasetId, granularity); + await checkSupervisorAvailability(datasourceRef) const existingDatasources = await getDatasourceListFromDruid(); if (!_.includes(existingDatasources.data, datasourceRef)) { From 6d83507003dfc6b867f079635cf4d6c59380195c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 11 Sep 2024 17:10:12 +0530 Subject: [PATCH 132/311] #OBS-I77 : fix: test cases --- .../src/controllers/DataOut/QueryValidator.ts | 2 +- .../DataOutTest/DataQueryTest.spec.ts | 36 +++++++++++++++++++ .../TemplateQuerying/TemplateQuerying.spec.ts | 11 ++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 9058603e..9d79069d 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -174,7 +174,7 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { } const checkSupervisorAvailability = async (datasourceRef: string) => { - const { data } = await druidHttpService.get('/druid/coordinator/v1/loadstatus'); + const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); const datasourceLoad = _.get(data, datasourceRef) if (!(datasourceLoad && datasourceLoad === 100)) { throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not fully available to query", "RANGE_NOT_SATISFIABLE", 416) diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 9f5cf5b3..b7af9bbb 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -7,6 +7,7 @@ import { config } from "../../../configs/Config"; import chaiSpies from "chai-spies" import { describe, it } from "mocha"; import { Datasource } from "../../../models/Datasource"; +import { druidHttpService } from "../../../connections/druidConnection"; chai.use(chaiSpies) chai.should(); chai.use(chaiHttp); @@ -33,6 +34,11 @@ describe("QUERY API TESTS", () => { response ) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["telemetry-events.1_rollup"]) @@ -74,6 +80,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -100,6 +111,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -126,6 +142,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -153,6 +174,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -180,6 +206,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -223,6 +254,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) diff --git a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts index 8da1818b..55368903 100644 --- a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts +++ b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts @@ -8,6 +8,7 @@ import { Datasource } from "../../../models/Datasource"; import nock from "nock"; import { config } from "../../../configs/Config"; import { templateQueryApiFixtures } from "./Fixtures"; +import { druidHttpService } from "../../../connections/druidConnection"; const apiId = "api.query.template.query"; const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" @@ -50,6 +51,11 @@ describe("QUERY TEMPLATE API", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_month": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_month"]) @@ -93,6 +99,11 @@ describe("QUERY TEMPLATE API", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_month": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_month"]) From e76e473b735580b48f4629a416b03196554f4944 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 17:22:38 +0530 Subject: [PATCH 133/311] #OBS-I165: corrections userRole access in notification --- .../src/controllers/NotificationChannel/Notification.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 949c8670..bc470c36 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,7 +12,7 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; _.set(payload, "created_by", userRole); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); @@ -34,7 +34,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; _.set(updatedPayload, "updated_by", userRole); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); @@ -78,7 +78,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -96,7 +96,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { From 9096b75cc602afe0e7434a6f9fe1cc203721d54b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 18:20:38 +0530 Subject: [PATCH 134/311] #OBS-I165: added operations_admin role and updated permissions --- .../src/middlewares/RBAC_middleware.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 92d352c8..e37bd916 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -5,6 +5,7 @@ import { config } from "../configs/Config"; import _ from "lodash"; enum roles { + OperationsAdmin = "operations_admin", Admin = "admin", DatasetManager = "dataset_manager", Viewer = "viewer", @@ -13,6 +14,28 @@ enum roles { } enum permissions { + AlertCreate = "api.alert.create", + AlertPublish = "api.alert.publish", + AlertUpdate = "api.alert.update", + AlertList = "api.alert.list", + AlertDelete = "api.alert.delete", + AlertDetails = "api.alert.getAlertDetails", + MetricCreate = "api.metric.add", + MetricList = "api.metric.list", + MetricUpdate = "api.metric.update", + MetricRemove = "api.metric.remove", + SilenceCreate = "api.alert.silence.create", + SilenceList = "api.alert.silence.list", + SilenceRead = "api.alert.silence.get", + SilenceEdit = "api.alert.silence.edit", + SilenceDelete = "api.alert.silence.delete", + NotificationChannelCreate = "api.alert.notification.create", + NotificationChannelList = "api.alert.notification.list", + NotificationChannelPublish = "api.alert.notification.publish", + NotificationChannelTest = "api.alert.notification.test", + NotificationChannelUpdate = "api.alert.notification.update", + NotificationChannelRetire = "api.alert.notification.retire", + NotificationChannelRead = "api.alert.notification.get", DatasetCreate = "api.datasets.create", DatasetUpdate = "api.datasets.update", DatasetRead = "api.datasets.read", @@ -54,6 +77,13 @@ const accessControl: AccessControl = { permissions.SQLQuery, permissions.DataOut, permissions.DataExhaust, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, ], [roles.DatasetCreator]: [ permissions.DatasetList, @@ -74,6 +104,13 @@ const accessControl: AccessControl = { permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, ], [roles.DatasetManager]: [ permissions.DatasetList, @@ -96,6 +133,13 @@ const accessControl: AccessControl = { permissions.DatasetSchemaGenerator, permissions.DatasetReset, permissions.DatasetStatusTransition, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, ], [roles.Admin]: [ permissions.DatasetCreate, @@ -119,6 +163,44 @@ const accessControl: AccessControl = { permissions.DatasetSchemaGenerator, permissions.DatasetReset, permissions.DatasetStatusTransition, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, + ], + [roles.OperationsAdmin]: [ + permissions.AlertCreate, + permissions.AlertPublish, + permissions.AlertUpdate, + permissions.AlertList, + permissions.AlertDelete, + permissions.AlertDetails, + permissions.MetricCreate, + permissions.MetricList, + permissions.MetricUpdate, + permissions.MetricRemove, + permissions.SilenceCreate, + permissions.SilenceList, + permissions.SilenceRead, + permissions.SilenceEdit, + permissions.SilenceDelete, + permissions.NotificationChannelCreate, + permissions.NotificationChannelList, + permissions.NotificationChannelPublish, + permissions.NotificationChannelTest, + permissions.NotificationChannelUpdate, + permissions.NotificationChannelRetire, + permissions.NotificationChannelRead, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, ], }; From 45adebe6f5af5afc913905f3a431991cdecf497b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 18:21:15 +0530 Subject: [PATCH 135/311] #OBS-I165: added rbac middleware for alert routers --- api-service/src/routes/AlertsRouter.ts | 49 +++++++++++++------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts index 01c843dd..6044e9fe 100644 --- a/api-service/src/routes/AlertsRouter.ts +++ b/api-service/src/routes/AlertsRouter.ts @@ -4,38 +4,39 @@ import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; import customAlertHandler from "../controllers/Alerts/Alerts"; import metricAliasHandler from "../controllers/Alerts/Metric"; import silenceHandler from "../controllers/Alerts/Silence"; +import checkRBAC from "../middlewares/RBAC_middleware"; export const alertsRouter = express.Router(); // Notifications -alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), notificationHandler.listHandler); -alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), notificationHandler.createHandler); -alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), notificationHandler.publishHandler); -alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), notificationHandler.testNotifationChannelHandler); -alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), notificationHandler.updateHandler); -alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), notificationHandler.retireHandler); -alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), notificationHandler.fetchHandler); +alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), checkRBAC.handler(), notificationHandler.listHandler); +alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), checkRBAC.handler(), notificationHandler.createHandler); +alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), checkRBAC.handler(), notificationHandler.publishHandler); +alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), checkRBAC.handler(), notificationHandler.testNotifationChannelHandler); +alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), checkRBAC.handler(), notificationHandler.updateHandler); +alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), checkRBAC.handler(), notificationHandler.retireHandler); +alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), checkRBAC.handler(), notificationHandler.fetchHandler); // alerts -alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), customAlertHandler.createAlertHandler); -alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), customAlertHandler.publishAlertHandler); -alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), customAlertHandler.searchAlertHandler); -alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), customAlertHandler.alertDetailsHandler); -alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteAlertHandler); -alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteSystemAlertsHandler); -alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), customAlertHandler.updateAlertHandler); +alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), checkRBAC.handler(), customAlertHandler.createAlertHandler); +alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), checkRBAC.handler(), customAlertHandler.publishAlertHandler); +alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), checkRBAC.handler(), customAlertHandler.searchAlertHandler); +alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), checkRBAC.handler(), customAlertHandler.alertDetailsHandler); +alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteAlertHandler); +alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteSystemAlertsHandler); +alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), checkRBAC.handler(), customAlertHandler.updateAlertHandler); // metrics -alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), metricAliasHandler.createMetricHandler); -alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), metricAliasHandler.listMetricsHandler); -alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"),metricAliasHandler.updateMetricHandler); -alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"),metricAliasHandler.deleteMetricHandler); -alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), metricAliasHandler.deleteMultipleMetricHandler); +alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), checkRBAC.handler(), metricAliasHandler.createMetricHandler); +alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), checkRBAC.handler(), metricAliasHandler.listMetricsHandler); +alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"), checkRBAC.handler(), metricAliasHandler.updateMetricHandler); +alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMetricHandler); +alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMultipleMetricHandler); // silence -alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"),silenceHandler.createHandler); -alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"),silenceHandler.listHandler); -alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"),silenceHandler.fetchHandler); -alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"),silenceHandler.updateHandler); -alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"),silenceHandler.deleteHandler); \ No newline at end of file +alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"), checkRBAC.handler(), silenceHandler.createHandler); +alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"), checkRBAC.handler(), silenceHandler.listHandler); +alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"), checkRBAC.handler(), silenceHandler.fetchHandler); +alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"), checkRBAC.handler(), silenceHandler.updateHandler); +alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"), checkRBAC.handler(), silenceHandler.deleteHandler); \ No newline at end of file From 02bd46b7219a73735ea84c0296dc27960721ae2f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 10:29:57 +0530 Subject: [PATCH 136/311] #OBS-I165: modified the user permissions into a json object --- .../src/middlewares/RBAC_middleware.ts | 346 ++++++++---------- 1 file changed, 147 insertions(+), 199 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index e37bd916..f9f615c6 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -4,205 +4,144 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; import _ from "lodash"; -enum roles { - OperationsAdmin = "operations_admin", - Admin = "admin", - DatasetManager = "dataset_manager", - Viewer = "viewer", - DatasetCreator = "dataset_creator", - Ingestor = "ingestor", -} - -enum permissions { - AlertCreate = "api.alert.create", - AlertPublish = "api.alert.publish", - AlertUpdate = "api.alert.update", - AlertList = "api.alert.list", - AlertDelete = "api.alert.delete", - AlertDetails = "api.alert.getAlertDetails", - MetricCreate = "api.metric.add", - MetricList = "api.metric.list", - MetricUpdate = "api.metric.update", - MetricRemove = "api.metric.remove", - SilenceCreate = "api.alert.silence.create", - SilenceList = "api.alert.silence.list", - SilenceRead = "api.alert.silence.get", - SilenceEdit = "api.alert.silence.edit", - SilenceDelete = "api.alert.silence.delete", - NotificationChannelCreate = "api.alert.notification.create", - NotificationChannelList = "api.alert.notification.list", - NotificationChannelPublish = "api.alert.notification.publish", - NotificationChannelTest = "api.alert.notification.test", - NotificationChannelUpdate = "api.alert.notification.update", - NotificationChannelRetire = "api.alert.notification.retire", - NotificationChannelRead = "api.alert.notification.get", - DatasetCreate = "api.datasets.create", - DatasetUpdate = "api.datasets.update", - DatasetRead = "api.datasets.read", - DatasetList = "api.datasets.list", - DataIngest = "api.data.in", - DataOut = "api.data.out", - DataExhaust = "api.data.exhaust", - QueryTemplateCreate = "api.query.template.create", - QueryTemplateRead = "api.query.template.read", - QueryTemplateDelete = "api.query.template.delete", - QueryTemplateList = "api.query.template.list", - QueryTemplateUpdate = "api.query.template.update", - QueryTemplate = "api.query.template.query", - SchemaValidator = "api.schema.validator", - GenerateSignedUrl = "api.files.generate-url", - DatasetStatusTransition = "api.datasets.status-transition", - DatasetHealth = "api.dataset.health", - DatasetReset = "api.dataset.reset", - DatasetSchemaGenerator = "api.datasets.dataschema", - DatasetExport = "api.datasets.export", - DatasetCopy = "api.datasets.copy", - ConnectorList = "api.connectors.list", - ConnectorRead = "api.connectors.read", - DatasetImport = "api.datasets.import", - SQLQuery = "api.obsrv.data.sql-query", -} - interface AccessControl { - [key: string]: string[]; + apiGroups : { + [key: string]: string[]; + }, + roles : { + [key: string]: string[]; + } } const accessControl: AccessControl = { - [roles.Ingestor]: [permissions.DataIngest], - [roles.Viewer]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.DatasetCreator]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.QueryTemplateUpdate, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.DatasetManager]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.QueryTemplateUpdate, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.Admin]: [ - permissions.DatasetCreate, - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.QueryTemplateUpdate, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.OperationsAdmin]: [ - permissions.AlertCreate, - permissions.AlertPublish, - permissions.AlertUpdate, - permissions.AlertList, - permissions.AlertDelete, - permissions.AlertDetails, - permissions.MetricCreate, - permissions.MetricList, - permissions.MetricUpdate, - permissions.MetricRemove, - permissions.SilenceCreate, - permissions.SilenceList, - permissions.SilenceRead, - permissions.SilenceEdit, - permissions.SilenceDelete, - permissions.NotificationChannelCreate, - permissions.NotificationChannelList, - permissions.NotificationChannelPublish, - permissions.NotificationChannelTest, - permissions.NotificationChannelUpdate, - permissions.NotificationChannelRetire, - permissions.NotificationChannelRead, - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - ], -}; + "apiGroups": { + "general_access": [ + "api.datasets.list", + "api.datasets.read", + "api.datasets.export", + "api.data.out", + "api.data.exhaust", + "api.alert.list", + "api.alert.getAlertDetails", + "api.metric.list", + "api.alert.silence.list", + "api.alert.silence.get", + "api.alert.notification.list", + "api.alert.notification.get" + ], + "restricted_dataset_api": [ + "api.datasets.reset", + "api.datasets.status-transition" + ], + "alert": [ + "api.alert.create", + "api.alert.publish", + "api.alert.update", + "api.alert.delete" + ], + "metric": [ + "api.metric.add", + "api.metric.update", + "api.metric.remove" + ], + "silence": [ + "api.alert.silence.create", + "api.alert.silence.edit", + "api.alert.silence.delete" + ], + "notificationChannel": [ + "api.alert.notification.create", + "api.alert.notification.publish", + "api.alert.notification.test", + "api.alert.notification.update", + "api.alert.notification.retire" + ], + "dataset": [ + "api.datasets.create", + "api.datasets.update", + "api.datasets.import", + "api.datasets.copy", + "api.dataset.health", + "api.datasets.dataschema" + ], + "data": [ + "api.data.in" + ], + "queryTemplate": [ + "api.query.template.create", + "api.query.template.read", + "api.query.template.delete", + "api.query.template.update", + "api.query.template.query", + "api.query.template.list" + ], + "schema": [ + "api.schema.validator" + ], + "file": [ + "api.files.generate-url" + ], + "connector": [ + "api.connectors.list", + "api.connectors.read" + ], + "sqlQuery": [ + "api.obsrv.data.sql-query" + ] + }, + "roles": { + "ingestor": [ + "data" + ], + "viewer": [ + "general_access", + "connector", + "sqlQuery" + ], + "dataset_creator": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery" + ], + "dataset_manager": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "admin": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "operations_admin": [ + "alert", + "metric", + "silence", + "notificationChannel", + "general_access" + ] + } +} export default { name: "rbac:middleware", @@ -239,10 +178,19 @@ export default { if (decoded && _.isObject(decoded)) { (req as any).userInfo = decoded; const action = (req as any).id; - const hasAccess = decoded?.roles?.some( - (role: string) => - accessControl[role] && accessControl[role].includes(action) - ); + // const hasAccess = decoded?.roles?.some( + // (role: string) => + // accessControl[role] && accessControl[role].includes(action) + // ); + const hasAccess = decoded?.roles?.some((role: string) => { + const apiGroups = accessControl.roles[role]; + + if (!apiGroups) return false; + + return apiGroups.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); if (!hasAccess) { return ResponseHandler.errorResponse( { From 998a846cbef048ebf4144b84c0f5de6131a21c0b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 11:14:33 +0530 Subject: [PATCH 137/311] #OBS-I165: removed unused code --- api-service/src/middlewares/RBAC_middleware.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index f9f615c6..1fc951ac 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -178,10 +178,6 @@ export default { if (decoded && _.isObject(decoded)) { (req as any).userInfo = decoded; const action = (req as any).id; - // const hasAccess = decoded?.roles?.some( - // (role: string) => - // accessControl[role] && accessControl[role].includes(action) - // ); const hasAccess = decoded?.roles?.some((role: string) => { const apiGroups = accessControl.roles[role]; From 9dc9e492ff42529dd2b9f337c121ea49df775491 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 12 Sep 2024 12:33:18 +0530 Subject: [PATCH 138/311] #OBS-I203: Remove the archived and purged status transition from the API --- .../DatasetStatusTransition.ts | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index c2a18021..c97b19c5 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -18,11 +18,9 @@ const allowedTransitions: Record = { Delete: [DatasetStatus.Draft, DatasetStatus.ReadyToPublish], ReadyToPublish: [DatasetStatus.Draft], Live: [DatasetStatus.ReadyToPublish], - Retire: [DatasetStatus.Live], - Archive: [DatasetStatus.Retired], - Purge: [DatasetStatus.Archived] + Retire: [DatasetStatus.Live] } -const liveDatasetActions = ["Retire", "Archive", "Purge"] +const liveDatasetActions = ["Retire"] const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) @@ -70,12 +68,8 @@ const datasetStatusTransition = async (req: Request, res: Response) => { case "Retire": await retireDataset(dataset); break; - case "Archive": - await archiveDataset(dataset); - break; - case "Purge": - await purgeDataset(dataset); - break; + default: + throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); @@ -239,14 +233,4 @@ export const restartPipeline = async (dataset: Record) => { return executeCommand(dataset.id, "RESTART_PIPELINE") } -const archiveDataset = async (dataset: Record) => { - - throw obsrvError(dataset.id, "ARCHIVE_NOT_IMPLEMENTED", "Archive functionality is not implemented", "NOT_IMPLEMENTED", 501) -} - -const purgeDataset = async (dataset: Record) => { - - throw obsrvError(dataset.id, "PURGE_NOT_IMPLEMENTED", "Purge functionality is not implemented", "NOT_IMPLEMENTED", 501) -} - export default datasetStatusTransition; \ No newline at end of file From a2e8a24b562e7eb8c7a74e609a4945f0002a4e0f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:07:04 +0530 Subject: [PATCH 139/311] #OBS-I165: added user permissions json --- .../src/middlewares/userPermissions.json | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 api-service/src/middlewares/userPermissions.json diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json new file mode 100644 index 00000000..400e218f --- /dev/null +++ b/api-service/src/middlewares/userPermissions.json @@ -0,0 +1,129 @@ +{ + "apiGroups": { + "general_access": [ + "api.datasets.list", + "api.datasets.read", + "api.datasets.export", + "api.data.out", + "api.data.exhaust", + "api.alert.list", + "api.alert.getAlertDetails", + "api.metric.list", + "api.alert.silence.list", + "api.alert.silence.get", + "api.alert.notification.list", + "api.alert.notification.get" + ], + "restricted_dataset_api": [ + "api.datasets.reset", + "api.datasets.status-transition" + ], + "alert": [ + "api.alert.create", + "api.alert.publish", + "api.alert.update", + "api.alert.delete" + ], + "metric": [ + "api.metric.add", + "api.metric.update", + "api.metric.remove" + ], + "silence": [ + "api.alert.silence.create", + "api.alert.silence.edit", + "api.alert.silence.delete" + ], + "notificationChannel": [ + "api.alert.notification.create", + "api.alert.notification.publish", + "api.alert.notification.test", + "api.alert.notification.update", + "api.alert.notification.retire" + ], + "dataset": [ + "api.datasets.create", + "api.datasets.update", + "api.datasets.import", + "api.datasets.copy", + "api.dataset.health", + "api.datasets.dataschema" + ], + "data": [ + "api.data.in" + ], + "queryTemplate": [ + "api.query.template.create", + "api.query.template.read", + "api.query.template.delete", + "api.query.template.update", + "api.query.template.query", + "api.query.template.list" + ], + "schema": [ + "api.schema.validator" + ], + "file": [ + "api.files.generate-url" + ], + "connector": [ + "api.connectors.list", + "api.connectors.read" + ], + "sqlQuery": [ + "api.obsrv.data.sql-query" + ] + }, + "roles": { + "ingestor": [ + "data" + ], + "viewer": [ + "general_access", + "connector", + "sqlQuery" + ], + "dataset_creator": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery" + ], + "dataset_manager": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "admin": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "operations_admin": [ + "alert", + "metric", + "silence", + "notificationChannel", + "general_access" + ] + } +} \ No newline at end of file From 179e8dcdcc6e31ce6a9213470477ec42d0d49a59 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:07:43 +0530 Subject: [PATCH 140/311] #OBS-I165: removed user roles for type --- api-service/src/services/telemetry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index f24995a9..b419db2a 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -18,7 +18,7 @@ const getDefaults = (userInfo:any) => { mid: v4(), actor: { id: userInfo?.id || "SYSTEM", - type: userInfo?.roles[0] || "User", + type: "User", }, context: { env, From 6a0bc426c2cf55ce1e738b8051f46b3556d91f40 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:09:13 +0530 Subject: [PATCH 141/311] #OBS-I165: added userID to req object and importing permissions from jsonfile --- .../src/middlewares/RBAC_middleware.ts | 134 +----------------- 1 file changed, 3 insertions(+), 131 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 1fc951ac..60e1ab72 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -3,7 +3,7 @@ import jwt from "jsonwebtoken"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; import _ from "lodash"; - +import userPermissions from "./userPermissions.json"; interface AccessControl { apiGroups : { [key: string]: string[]; @@ -13,135 +13,7 @@ interface AccessControl { } } -const accessControl: AccessControl = { - "apiGroups": { - "general_access": [ - "api.datasets.list", - "api.datasets.read", - "api.datasets.export", - "api.data.out", - "api.data.exhaust", - "api.alert.list", - "api.alert.getAlertDetails", - "api.metric.list", - "api.alert.silence.list", - "api.alert.silence.get", - "api.alert.notification.list", - "api.alert.notification.get" - ], - "restricted_dataset_api": [ - "api.datasets.reset", - "api.datasets.status-transition" - ], - "alert": [ - "api.alert.create", - "api.alert.publish", - "api.alert.update", - "api.alert.delete" - ], - "metric": [ - "api.metric.add", - "api.metric.update", - "api.metric.remove" - ], - "silence": [ - "api.alert.silence.create", - "api.alert.silence.edit", - "api.alert.silence.delete" - ], - "notificationChannel": [ - "api.alert.notification.create", - "api.alert.notification.publish", - "api.alert.notification.test", - "api.alert.notification.update", - "api.alert.notification.retire" - ], - "dataset": [ - "api.datasets.create", - "api.datasets.update", - "api.datasets.import", - "api.datasets.copy", - "api.dataset.health", - "api.datasets.dataschema" - ], - "data": [ - "api.data.in" - ], - "queryTemplate": [ - "api.query.template.create", - "api.query.template.read", - "api.query.template.delete", - "api.query.template.update", - "api.query.template.query", - "api.query.template.list" - ], - "schema": [ - "api.schema.validator" - ], - "file": [ - "api.files.generate-url" - ], - "connector": [ - "api.connectors.list", - "api.connectors.read" - ], - "sqlQuery": [ - "api.obsrv.data.sql-query" - ] - }, - "roles": { - "ingestor": [ - "data" - ], - "viewer": [ - "general_access", - "connector", - "sqlQuery" - ], - "dataset_creator": [ - "general_access", - "connector", - "sqlQuery", - "dataset", - "queryTemplate", - "schema", - "file", - "connector", - "sqlQuery" - ], - "dataset_manager": [ - "general_access", - "connector", - "sqlQuery", - "dataset", - "queryTemplate", - "schema", - "file", - "connector", - "sqlQuery", - "restricted_dataset_api" - ], - "admin": [ - "general_access", - "connector", - "sqlQuery", - "dataset", - "queryTemplate", - "schema", - "file", - "connector", - "sqlQuery", - "restricted_dataset_api" - ], - "operations_admin": [ - "alert", - "metric", - "silence", - "notificationChannel", - "general_access" - ] - } -} +const accessControl: AccessControl = userPermissions; export default { name: "rbac:middleware", @@ -176,7 +48,7 @@ export default { ); } if (decoded && _.isObject(decoded)) { - (req as any).userInfo = decoded; + (req as any).userID = decoded?.id; const action = (req as any).id; const hasAccess = decoded?.roles?.some((role: string) => { const apiGroups = accessControl.roles[role]; From 52273e9f7f5a50a35e33eabeb38ee16cd609144b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:11:12 +0530 Subject: [PATCH 142/311] #OBS-I165: handled rbac disabled scenario and updated userID instead of userRole --- api-service/src/controllers/Alerts/Alerts.ts | 13 ++++++++----- api-service/src/controllers/Alerts/Silence.ts | 4 ++-- .../CreateQueryTemplate/CreateTemplateController.ts | 3 ++- .../src/controllers/DatasetCopy/DatasetCopy.ts | 2 +- .../src/controllers/DatasetCreate/DatasetCreate.ts | 2 +- .../src/controllers/DatasetImport/DatasetImport.ts | 2 +- .../src/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 2 +- .../src/controllers/DatasetUpdate/DatasetUpdate.ts | 3 ++- .../controllers/NotificationChannel/Notification.ts | 8 ++++---- .../UpdateQueryTemplate/UpdateTemplateController.ts | 3 ++- 11 files changed, 25 insertions(+), 19 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 131fc9a2..90142188 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,7 +13,8 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); - _.set(alertPayload, "created_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(alertPayload, "created_by", userRole); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -31,7 +32,8 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); - _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userRole); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -89,7 +91,8 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); - _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userRole); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -106,13 +109,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); + const userRole = (req as any)?.userID || "SYSTEM"; if (rulePayload.status == "live") { - _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); + _.set(rulePayload, "updated_by", userRole); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - const userRole = (req as any)?.userInfo?.roles[0]; await Alert.update({ ...updatedPayload, status: "draft", updated_by: userRole }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index fd2bee31..f0ae97d0 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,7 +20,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, @@ -80,7 +80,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 94b4cffd..2a7d861c 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,7 +42,8 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); - _.set(data, "created_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(data, "created_by", userRole); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 8ddc9fe2..0d803ee0 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,7 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); - const userRole = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; _.set(dataset, "created_by", userRole); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 2e476b7c..550413b1 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,7 +34,7 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) - const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; + const userRole = (req as any)?.userID || "SYSTEM"; _.set(draftDataset, "created_by", userRole); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 97314ecc..6ff8f09d 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,7 +15,7 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const userRole = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; _.set(datasetPayload, "created_by", userRole); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 5e3a869c..49487014 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,7 +30,7 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; - const userRole = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userRole) : await readDataset(dataset_id, attributes) if (!dataset) { diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index eb03c7b5..d5ce4ee8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,7 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) - const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; + const userRole = (req as any)?.userID || "SYSTEM"; validateDataset(dataset, dataset_id, status); switch (status) { diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 679b3fcc..d83962fa 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,7 +59,8 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - _.set(draftDataset, "updated_by", (req as any)?.userInfo?.roles[0] ) + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(draftDataset, "updated_by", userRole ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index bc470c36..21d4ae74 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,7 +12,7 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; _.set(payload, "created_by", userRole); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); @@ -34,7 +34,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; _.set(updatedPayload, "updated_by", userRole); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); @@ -78,7 +78,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -96,7 +96,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 30f3c5c5..741b2114 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,8 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - requestBody.request.updated_by = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; + requestBody.request.updated_by = userRole; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); From 2eac48fc4308d3437861afa6a287c2d63c06c348 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 18:01:08 +0530 Subject: [PATCH 143/311] #OBS-I165: changed the userRole to userID --- api-service/src/controllers/Alerts/Alerts.ts | 18 +++++++++--------- api-service/src/controllers/Alerts/Silence.ts | 8 ++++---- .../CreateTemplateController.ts | 4 ++-- .../src/controllers/DatasetCopy/DatasetCopy.ts | 4 ++-- .../controllers/DatasetCreate/DatasetCreate.ts | 4 ++-- .../controllers/DatasetImport/DatasetImport.ts | 10 +++++----- .../src/controllers/DatasetRead/DatasetRead.ts | 10 +++++----- .../DatasetStatusTransition.ts | 12 ++++++------ .../controllers/DatasetUpdate/DatasetUpdate.ts | 4 ++-- .../NotificationChannel/Notification.ts | 16 ++++++++-------- .../UpdateTemplateController.ts | 4 ++-- api-service/src/services/DatasetService.ts | 8 ++++---- api-service/src/services/telemetry.ts | 6 +++--- 13 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 90142188..8e749e22 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,8 +13,8 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(alertPayload, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(alertPayload, "created_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -32,8 +32,8 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(rulePayload, "updated_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -91,8 +91,8 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(rulePayload, "updated_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -109,14 +109,14 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); - const userRole = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID || "SYSTEM"; if (rulePayload.status == "live") { - _.set(rulePayload, "updated_by", userRole); + _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: "draft", updated_by: userRole }, { where: { id: alertId } }); + await Alert.update({ ...updatedPayload, status: "draft", updated_by: userID }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index f0ae97d0..4a10730a 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,14 +20,14 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); - const userRole = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID || "SYSTEM"; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, alert_id: alertId, start_time: start_date, end_time: end_date, - created_by : userRole, + created_by : userID, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); @@ -80,12 +80,12 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); - const userRole = (request as any)?.userID; + const userID = (request as any)?.userID; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, end_time: updatedEndTime, - updated_by: userRole, + updated_by: userID, } const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 2a7d861c..d112bad1 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,8 +42,8 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(data, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(data, "created_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 0d803ee0..8bdf3796 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,8 +40,8 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(dataset, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(dataset, "created_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 550413b1..e91fb59d 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,8 +34,8 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(draftDataset, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(draftDataset, "created_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 6ff8f09d..ad5b5df8 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,21 +15,21 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(datasetPayload, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(datasetPayload, "created_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await importDataset(updatedDataset, overwrite, userRole); + const dataset = await importDataset(updatedDataset, overwrite, userID); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const importDataset = async (dataset: Record, overwrite: string | any, userRole : string) => { +const importDataset = async (dataset: Record, overwrite: string | any, userID : string) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { - _.set(dataset, "updated_by", userRole); + _.set(dataset, "updated_by", userID); const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 49487014..3de9ab6e 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,9 +30,9 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; - const userRole = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID || "SYSTEM"; const attributes = !fields ? defaultFields : _.split(fields, ","); - const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userRole) : await readDataset(dataset_id, attributes) + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userID) : await readDataset(dataset_id, attributes) if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } @@ -42,19 +42,19 @@ const datasetRead = async (req: Request, res: Response) => { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } -const readDraftDataset = async (datasetId: string, attributes: string[], userRole: string): Promise => { +const readDraftDataset = async (datasetId: string, attributes: string[], userID: string): Promise => { const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); - const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userRole) + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userID) return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if (liveDataset) { - const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userRole) + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userID) return _.pick(dataset, attributes); } diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index d5ce4ee8..041bfd60 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,7 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) - const userRole = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID || "SYSTEM"; validateDataset(dataset, dataset_id, status); switch (status) { @@ -63,13 +63,13 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await deleteDataset(dataset); break; case "ReadyToPublish": - await readyForPublish(dataset, userRole); + await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userRole); + await publishDataset(dataset, userID); break; case "Retire": - await retireDataset(dataset, userRole); + await retireDataset(dataset, userID); break; case "Archive": await archiveDataset(dataset); @@ -134,10 +134,10 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userRole: any) => { +const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - _.set(draftDataset, ["updated_by"], userRole); + _.set(draftDataset, ["updated_by"], userID); console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index d83962fa..16c9d9a3 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,8 +59,8 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(draftDataset, "updated_by", userRole ) + const userID = (req as any)?.userID || "SYSTEM"; + _.set(draftDataset, "updated_by", userID ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 21d4ae74..5f8e66f1 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,8 +12,8 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userRole = (request as any)?.userID || "SYSTEM"; - _.set(payload, "created_by", userRole); + const userID = (request as any)?.userID || "SYSTEM"; + _.set(payload, "created_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) @@ -34,8 +34,8 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userRole = (request as any)?.userID || "SYSTEM"; - _.set(updatedPayload, "updated_by", userRole); + const userID = (request as any)?.userID || "SYSTEM"; + _.set(updatedPayload, "updated_by", userID); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -78,8 +78,8 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userRole = (request as any)?.userID || "SYSTEM"; - await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) + const userID = (request as any)?.userID || "SYSTEM"; + await Notification.update({ status: "retired", updated_by: userID }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) @@ -96,8 +96,8 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userRole = (request as any)?.userID || "SYSTEM"; - Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); + const userID = (request as any)?.userID || "SYSTEM"; + Notification.update({ status: "live", updated_by: userID }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 741b2114..7f7775ae 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,8 +38,8 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - const userRole = (req as any)?.userID || "SYSTEM"; - requestBody.request.updated_by = userRole; + const userID = (req as any)?.userID || "SYSTEM"; + requestBody.request.updated_by = userID; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 4bb27ed7..e7bda7d9 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -92,10 +92,10 @@ class DatasetService { return responseData; } - migrateDraftDataset = async (datasetId: string, dataset: Record, userRole: string): Promise => { + migrateDraftDataset = async (datasetId: string, dataset: Record, userID: string): Promise => { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); - _.set(draftDataset, "updated_by", userRole); + _.set(draftDataset, "updated_by", userID); const transaction = await sequelize.transaction(); try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); @@ -167,7 +167,7 @@ class DatasetService { } } - createDraftDatasetFromLive = async (dataset: Model, userRole: string) => { + createDraftDatasetFromLive = async (dataset: Model, userID: string) => { const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config: any = _.get(dataset, "dataset_config"); @@ -233,7 +233,7 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft - draftDataset["created_by"] = userRole; + draftDataset["created_by"] = userID; const result = await DatasetDraft.create(draftDataset); return _.get(result, "dataValues") } diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index b419db2a..8408dfb3 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -10,14 +10,14 @@ const telemetryTopic = _.get(appConfig, "telemetry_dataset"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const getDefaults = (userInfo:any) => { +const getDefaults = (userID:any) => { return { eid: "AUDIT", ets: Date.now(), ver: "1.0.0", mid: v4(), actor: { - id: userInfo?.id || "SYSTEM", + id: userID || "SYSTEM", type: "User", }, context: { @@ -128,7 +128,7 @@ export const processAuditEvents = (request: Request) => { _.set(auditEvent, "edata.transition.toState", toState); _.set(auditEvent, "edata.transition.fromState", fromState); } - const telemetryEvent = getDefaults((request as any)?.userInfo); + const telemetryEvent = getDefaults((request as any)?.userID); _.set(telemetryEvent, "edata", edata); _.set(telemetryEvent, "object", { ...(object.id && object.type && { ...object, ver: "1.0.0" }) }); sendTelemetryEvents(telemetryEvent); From 0b6c05dbf99e6dfa48d603c317d4d2c5fbd85da4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 19:30:33 +0530 Subject: [PATCH 144/311] #OBS-I165: added createdby for dataset publish api --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 041bfd60..debf430f 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -137,6 +137,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record + _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); From 6b68ad2aec357f96a5bd2bc3925bd32928cc34a9 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 16 Sep 2024 11:42:02 +0530 Subject: [PATCH 145/311] #OBS-I165: added error condition and modified status code --- .../src/middlewares/RBAC_middleware.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 60e1ab72..8b073fbe 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -27,8 +27,8 @@ export default { if (!token) { return ResponseHandler.errorResponse( { - statusCode: 403, - errCode: "FORBIDDEN", + statusCode: 401, + errCode: "Unauthorized access", message: "No token provided", }, req, @@ -48,6 +48,17 @@ export default { ); } if (decoded && _.isObject(decoded)) { + if (!decoded?.id) { + return ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "User ID is missing from the decoded token.", + }, + req, + res + ); + } (req as any).userID = decoded?.id; const action = (req as any).id; const hasAccess = decoded?.roles?.some((role: string) => { @@ -62,8 +73,8 @@ export default { if (!hasAccess) { return ResponseHandler.errorResponse( { - statusCode: 401, - errCode: "Unauthorized access", + statusCode: 403, + errCode: "FORBIDDEN", message: "Access denied for the user", }, req, From 904a526d7adb7726fbee219f53f7db87340ebd28 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 16 Sep 2024 11:42:53 +0530 Subject: [PATCH 146/311] #OBS-I165: removed redundant code --- api-service/src/controllers/Alerts/Alerts.ts | 6 +++--- api-service/src/controllers/Alerts/Silence.ts | 2 +- .../CreateQueryTemplate/CreateTemplateController.ts | 2 +- api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 2 +- .../src/controllers/DatasetCreate/DatasetCreate.ts | 2 +- .../src/controllers/DatasetImport/DatasetImport.ts | 2 +- api-service/src/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition/DatasetStatusTransition.ts | 2 +- .../src/controllers/DatasetUpdate/DatasetUpdate.ts | 2 +- .../src/controllers/NotificationChannel/Notification.ts | 8 ++++---- .../UpdateQueryTemplate/UpdateTemplateController.ts | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 8e749e22..eaaef01d 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,7 +13,7 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(alertPayload, "created_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); @@ -32,7 +32,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); @@ -109,7 +109,7 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; if (rulePayload.status == "live") { _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, false); diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 4a10730a..15a7dfda 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,7 +20,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index d112bad1..68e36a22 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,7 +42,7 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(data, "created_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 8bdf3796..28707bd2 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,7 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index e91fb59d..62b2e4b7 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,7 +34,7 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(draftDataset, "created_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index ad5b5df8..6ef907c2 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,7 +15,7 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(datasetPayload, "created_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 3de9ab6e..d428b02a 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,7 +30,7 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userID) : await readDataset(dataset_id, attributes) if (!dataset) { diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index debf430f..eff474fd 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,7 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; validateDataset(dataset, dataset_id, status); switch (status) { diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 16c9d9a3..7277b1b8 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,7 +59,7 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(draftDataset, "updated_by", userID ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 5f8e66f1..deeee234 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,7 +12,7 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; _.set(payload, "created_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); @@ -34,7 +34,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; _.set(updatedPayload, "updated_by", userID); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); @@ -78,7 +78,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; await Notification.update({ status: "retired", updated_by: userID }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -96,7 +96,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; Notification.update({ status: "live", updated_by: userID }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 7f7775ae..350b25f7 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,7 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; requestBody.request.updated_by = userID; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) From 0ddb49eca8beb979e3af57ba3318c21437162233 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 16 Sep 2024 11:43:50 +0530 Subject: [PATCH 147/311] #OBS-I222 : Resolved regexp route issue in express version 5 --- api-service/src/app.ts | 2 +- api-service/src/routes/DruidProxyRouter.ts | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 27b9fce6..6615ad74 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -22,7 +22,7 @@ app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); app.use("/", metricRouter); -// app.use("*", ResponseHandler.routeNotFound); +app.use(/(.*)/, ResponseHandler.routeNotFound); app.use(obsrvErrorHandler); app.listen(config.api_port, () => { diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index 70fa3ee6..fdf73315 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -9,15 +9,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; export const druidProxyRouter = express.Router(); // Send a 410 Gone response to all V1 API calls -// druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all(/\/datasets\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/dataset\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/datasources\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/data\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/template\/v1(.*)/, ResponseHandler.goneResponse); -// // Druid Proxy APIs for Metabase integration -// druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) -// druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) -// druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) -// druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -// druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +// Druid Proxy APIs for Metabase integration +druidProxyRouter.post(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative); +druidProxyRouter.get(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet); +druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) +druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file From 9183129b1134eeae423b536dbd22cf75759b3e3a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 16 Sep 2024 15:45:08 +0530 Subject: [PATCH 148/311] #OBS-I77 : query api changes to check avialability of datasource --- api-service/src/controllers/DataOut/QueryValidator.ts | 11 +++++++---- .../DataOutTest/DataQueryTest.spec.ts | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 9d79069d..1469e746 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -16,7 +16,7 @@ let dataset_id: string; let requestBody: any; let msgid: string; const errCode = { - notFound: "DATA_OUT_SOURCE_NOT_FOUND", + notFound: "DATASOURCE_NOT_FOUND", invalidDateRange: "DATA_OUT_INVALID_DATE_RANGE" } @@ -175,9 +175,12 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { const checkSupervisorAvailability = async (datasourceRef: string) => { const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); - const datasourceLoad = _.get(data, datasourceRef) - if (!(datasourceLoad && datasourceLoad === 100)) { - throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not fully available to query", "RANGE_NOT_SATISFIABLE", 416) + const datasourceAvailaibility = _.get(data, datasourceRef) + if (_.isUndefined(datasourceAvailaibility)) { + throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not available for querying", "NOT_FOUND", 404) + } + if (datasourceAvailaibility !== 100) { + throw obsrvError("", "DATASOURCE_NOT_FULLY_AVAILABLE", "Datasource not fully available for querying", "RANGE_NOT_SATISFIABLE", 416) } } diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index b7af9bbb..2827424b 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -52,7 +52,7 @@ describe("QUERY API TESTS", () => { res.body.params.msgid.should.be.eq(msgid); res.body.responseCode.should.be.eq("NOT_FOUND"); res.body.error.message.should.be.eq("Dataset telemetry-events with table week is not available for querying"); - res.body.error.code.should.be.eq("DATA_OUT_SOURCE_NOT_FOUND"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); done(); }); }); @@ -71,7 +71,7 @@ describe("QUERY API TESTS", () => { res.body.responseCode.should.be.eq("NOT_FOUND"); res.body.params.msgid.should.be.eq(msgid); res.body.error.message.should.be.eq("Datasource telemetry-events not available for querying"); - res.body.error.code.should.be.eq("DATA_OUT_SOURCE_NOT_FOUND"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); done(); }); }); From 31fbbefc535a42fc9de5931ec65e234b6296a683 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 16 Sep 2024 16:25:46 +0530 Subject: [PATCH 149/311] #OBS-I77 : fix: spell fix --- api-service/src/controllers/DataOut/QueryValidator.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 1469e746..af892572 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -175,11 +175,11 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { const checkSupervisorAvailability = async (datasourceRef: string) => { const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); - const datasourceAvailaibility = _.get(data, datasourceRef) - if (_.isUndefined(datasourceAvailaibility)) { + const datasourceAvailability = _.get(data, datasourceRef) + if (_.isUndefined(datasourceAvailability)) { throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not available for querying", "NOT_FOUND", 404) } - if (datasourceAvailaibility !== 100) { + if (datasourceAvailability !== 100) { throw obsrvError("", "DATASOURCE_NOT_FULLY_AVAILABLE", "Datasource not fully available for querying", "RANGE_NOT_SATISFIABLE", 416) } } From 63625bfa2a50548fa3ec0967cf8ccc1adcdef3f0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 16 Sep 2024 17:09:41 +0530 Subject: [PATCH 150/311] #OBS-I77 : granularity spell changes --- api-service/src/controllers/DataOut/QueryValidator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index af892572..45d99049 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -156,7 +156,7 @@ const validateQueryRules = (queryPayload: any, limits: any) => { : { message: "Invalid date range! the date range cannot be a null value", statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange }; }; -const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { +const getDataSourceRef = async (datasetId: string, requestGranularity?: string) => { const dataSources = await getDatasourceList(datasetId) if (_.isEmpty(dataSources)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Datasource ${datasetId} not available in datasource live table`, code: errCode.notFound }) @@ -168,7 +168,7 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { if (!aggregated) { return true; } - return aggregated && srcGranularity ? granularity === srcGranularity : false; + return aggregated && requestGranularity ? granularity === requestGranularity : false; }); return _.get(record, ["dataValues", "datasource_ref"]) } From fb3fb407d1f7c850576629c980e6cc1c9f99d7aa Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 11:04:17 +0530 Subject: [PATCH 151/311] #OBS-I77 : fix: test scenarios added --- .../src/controllers/DataOut/QueryValidator.ts | 3 + .../DataOutTest/DataQueryTest.spec.ts | 74 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 45d99049..4c47a90e 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -187,6 +187,9 @@ const checkSupervisorAvailability = async (datasourceRef: string) => { const setDatasourceRef = async (datasetId: string, payload: any): Promise => { const granularity = _.get(payload, "context.aggregationLevel") const datasourceRef = await getDataSourceRef(datasetId, granularity); + if (!datasourceRef) { + throw obsrvError("", "DATASOURCE_NOT_FOUND", "Datasource not found to query", "NOT_FOUND", 404) + } await checkSupervisorAvailability(datasourceRef) const existingDatasources = await getDatasourceListFromDruid(); diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 2827424b..ea905305 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -19,6 +19,7 @@ const nativeQueryEndpointDruid = config?.query_api?.druid?.native_query_path; const sqlQueryEndpoint = config?.query_api?.druid?.sql_query_path; const response = [{ dataValues: { datasource_ref: "test.1_rollup_week", metadata: { aggregated: true, granularity: "week" } } }] +const invalidResponse = [{ dataValues: { datasource_ref: "test.1_rollup_week", metadata: { aggregated: true, granularity: "n/a" } } }] const msgid = "e180ecac-8f41-4f21-9a21-0b3a1a368917"; describe("QUERY API TESTS", () => { @@ -76,6 +77,79 @@ describe("QUERY API TESTS", () => { }); }); + it("Query api failure: Datasource not available in druid", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + response + ) + }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "telemetry_events": 100 } + }) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(404); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("NOT_FOUND"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not available for querying"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_AVAILABLE"); + done(); + }); + }); + + it("Query api failure: Datasource not fully loaded in druid", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + response + ) + }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 20 } + }) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(416); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("RANGE_NOT_SATISFIABLE"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not fully available for querying"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FULLY_AVAILABLE"); + done(); + }); + }); + + it("Query api failure: Datasource not found", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + invalidResponse + ) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(404); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("NOT_FOUND"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not found to query"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); + done(); + }); + }); + it("Query api failure : when druid is down, it should raise error when native query endpoint is called", (done) => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) From bb60d4a9f3da48e56d661885a5cf92f612656c3c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 12:05:20 +0530 Subject: [PATCH 152/311] #OBS-I131 : fix: stop deleting draft records if submission tasks fail --- command-service/src/command/db_command.py | 17 +------------ command-service/src/command/druid_command.py | 24 ++++++++++++++++++- command-service/src/config/service_config.yml | 2 +- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 89db3392..20e9c527 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -48,7 +48,6 @@ def _change_dataset_to_active(self, command_payload: CommandPayload): self._insert_datasource_record(dataset_id, draft_dataset_id) self._insert_connector_instances(dataset_id, draft_dataset_record) self._insert_dataset_transformations(dataset_id, draft_dataset_record) - self._delete_draft_dataset(dataset_id, draft_dataset_id) return ActionResponse(status="OK", status_code=200) else: return ActionResponse( @@ -412,18 +411,4 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): """ result = self.db_service.execute_upsert(sql=insert_query, params=params) print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") - return result - - def _delete_draft_dataset(self, dataset_id, draft_dataset_id): - - self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft source config/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) - print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file + return result \ No newline at end of file diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index f43222b4..73d5bf86 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -34,6 +34,7 @@ def _submit_ingestion_task(self, dataset_id): print( f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." ) + task_submitted = 1 for record in datasources_records: if record["dataset_type"] == "event" and record["type"] == "druid": print(f"Submitting ingestion task for datasource ...") @@ -41,8 +42,15 @@ def _submit_ingestion_task(self, dataset_id): response = self.http_service.post( url=f"{self.router_url}/{self.supervisor_endpoint}", body=ingestion_spec, - headers={"Content-Type": "application/json"}, + headers={"Content-Type": "application/json"} ) + if response.status != 200: + task_submitted = 0 + break + if task_submitted: + query=f"SELECT id FROM datasets_draft WHERE dataset_id= %s" + response = self.db_service.execute_select_one(sql=query, params=(dataset_id,)) + self._delete_draft_dataset(dataset_id, response[0]) return ActionResponse(status="OK", status_code=200) else: print( @@ -51,3 +59,17 @@ def _submit_ingestion_task(self, dataset_id): return ActionResponse( status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" ) + + def _delete_draft_dataset(self, dataset_id, draft_dataset_id): + + self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft source config/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) + print(f"Draft Dataset for {dataset_id} is deleted successfully...") diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 2c432ba0..8b9f2bdc 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -117,7 +117,7 @@ config_service: port: 4000 druid: - router_host: localhost + router_host: http://localhost router_port: 8888 supervisor_endpoint: indexer/v1/supervisor From 586090e54606cb9d9834f2e978736283cf372235 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 12:42:25 +0530 Subject: [PATCH 153/311] #OBS-I131 : fix: upsert if republished --- api-service/src/services/DatasetService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 2e800212..d7679cbe 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -325,7 +325,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -334,7 +334,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -345,7 +345,7 @@ class DatasetService { const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private createDraftDatasource = (draftDataset: Record, type: string): Record => { From 8b560493cfe5e4c8f2c19fdb3fa35794cb87b035 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 12:43:55 +0530 Subject: [PATCH 154/311] #OBS-I131 : fix: test case fixes --- .../DatasetStatusTransition/DatasetLive.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index e26ddafe..38dced09 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -38,7 +38,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -80,7 +80,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -122,7 +122,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "api_version":"v2", "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) chai.spy.on(Datasource, "findOne", () => { @@ -289,7 +289,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { From aacd420ad602b22b6368eba98c2fc12f588db8bd Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 12:47:21 +0530 Subject: [PATCH 155/311] #OBS-I165: added default userID, modified error codes, added errorhandler function --- .../src/middlewares/RBAC_middleware.ts | 82 +++++++++++-------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 8b073fbe..77f9581f 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -15,49 +15,56 @@ interface AccessControl { const accessControl: AccessControl = userPermissions; +const errorHandler = (statusCode: number, message: string, req: Request, res: Response) => { + let errCode: string; + let code: string; + + switch (statusCode) { + case 401: + errCode = "Unauthorized access"; + code = "UNAUTHORIZED ACCESS"; + break; + case 403: + errCode = "Forbidden access"; + code = "FORBIDDEN ACCESS"; + break; + default: + errCode = "Unknown error"; + code = "UNKNOWN ERROR"; + } + + return ResponseHandler.errorResponse( + { + statusCode, + errCode, + message, + code, + }, + req, + res + ); +}; + export default { name: "rbac:middleware", handler: () => (req: Request, res: Response, next: NextFunction) => { try { if (_.lowerCase(config.is_RBAC_enabled) === "false") { + (req as any).userID = "SYSTEM"; next(); } else { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { - return ResponseHandler.errorResponse( - { - statusCode: 401, - errCode: "Unauthorized access", - message: "No token provided", - }, - req, - res - ); + errorHandler(401, "No token provided", req, res); } jwt.verify(token as string, public_key, (err, decoded) => { if (err) { - return ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "Token verification failed", - }, - req, - res - ); + errorHandler(401, "Token verification failed", req, res); } if (decoded && _.isObject(decoded)) { if (!decoded?.id) { - return ResponseHandler.errorResponse( - { - statusCode: 401, - errCode: "Unauthorized access", - message: "User ID is missing from the decoded token.", - }, - req, - res - ); + errorHandler(401, "User ID is missing from the decoded token.", req, res); } (req as any).userID = decoded?.id; const action = (req as any).id; @@ -71,15 +78,18 @@ export default { ); }); if (!hasAccess) { - return ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "Access denied for the user", - }, - req, - res - ); + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + if (!apiGroups) return false; + return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + const rolesMessage = rolesWithAccess.length > 0 + ? `Roles with access: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + + errorHandler(403, errorMessage, req, res); } next(); } From 6e5045291d17309b98f81b0717a2a4143878d0e0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 12:58:28 +0530 Subject: [PATCH 156/311] #OBS-I165: added updatedby by default when an entry created --- api-service/src/controllers/Alerts/Alerts.ts | 1 + api-service/src/controllers/Alerts/Silence.ts | 1 + .../controllers/CreateQueryTemplate/CreateTemplateController.ts | 1 + api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 1 + api-service/src/controllers/DatasetCreate/DatasetCreate.ts | 1 + api-service/src/controllers/DatasetImport/DatasetImport.ts | 1 + api-service/src/controllers/NotificationChannel/Notification.ts | 1 + 7 files changed, 7 insertions(+) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index eaaef01d..b3bda19e 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -15,6 +15,7 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio const alertPayload = getAlertPayload(req.body); const userID = (req as any)?.userID; _.set(alertPayload, "created_by", userID); + _.set(alertPayload, "updated_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 15a7dfda..f424465a 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -28,6 +28,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun start_time: start_date, end_time: end_date, created_by : userID, + updated_by : userID, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 68e36a22..c7fffa29 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -44,6 +44,7 @@ export const createQueryTemplate = async (req: Request, res: Response) => { const data = transformRequest(requestBody, templateName); const userID = (req as any)?.userID; _.set(data, "created_by", userID); + _.set(data, "updated_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 28707bd2..9015e9c7 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -42,6 +42,7 @@ const datasetCopy = async (req: Request, res: Response) => { const dataset = await fetchDataset(req); const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); + _.set(dataset, "updated_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 62b2e4b7..bb96dac2 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -36,6 +36,7 @@ const datasetCreate = async (req: Request, res: Response) => { const draftDataset = getDraftDataset(req.body.request) const userID = (req as any)?.userID; _.set(draftDataset, "created_by", userID); + _.set(draftDataset, "updated_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 6ef907c2..2d0312b5 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -17,6 +17,7 @@ const datasetImport = async (req: Request, res: Response) => { } const userID = (req as any)?.userID; _.set(datasetPayload, "created_by", userID); + _.set(datasetPayload, "updated_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index deeee234..d8850436 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -14,6 +14,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun const payload = request.body; const userID = (request as any)?.userID; _.set(payload, "created_by", userID); + _.set(payload, "updated_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) From cdb55a7fdcf10b43228ee0b7385645d508c5e230 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 12:59:28 +0530 Subject: [PATCH 157/311] #OBS-I165: removed redundant code --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index eff474fd..dfc13211 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -139,7 +139,6 @@ const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); - console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) From 45f9c617e36fcdd48776fd4d3bf85d150fd23ed1 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 15:17:36 +0530 Subject: [PATCH 158/311] #OBS-I165: modified errorhandler --- .../src/middlewares/RBAC_middleware.ts | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 77f9581f..c00199a5 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -4,6 +4,7 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; import _ from "lodash"; import userPermissions from "./userPermissions.json"; +import httpStatus from "http-status"; interface AccessControl { apiGroups : { [key: string]: string[]; @@ -16,22 +17,18 @@ interface AccessControl { const accessControl: AccessControl = userPermissions; const errorHandler = (statusCode: number, message: string, req: Request, res: Response) => { - let errCode: string; - let code: string; + const errorMapping: Record = { + 401: { + errCode: httpStatus["401_NAME"], + code: "UNAUTHORIZED ACCESS", + }, + 403: { + errCode: httpStatus["403_NAME"], + code: "FORBIDDEN ACCESS", + }, + }; - switch (statusCode) { - case 401: - errCode = "Unauthorized access"; - code = "UNAUTHORIZED ACCESS"; - break; - case 403: - errCode = "Forbidden access"; - code = "FORBIDDEN ACCESS"; - break; - default: - errCode = "Unknown error"; - code = "UNKNOWN ERROR"; - } + const { errCode, code } = errorMapping[statusCode]; return ResponseHandler.errorResponse( { @@ -56,15 +53,15 @@ export default { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { - errorHandler(401, "No token provided", req, res); + return errorHandler(401, "No token provided", req, res); } jwt.verify(token as string, public_key, (err, decoded) => { if (err) { - errorHandler(401, "Token verification failed", req, res); + return errorHandler(401, "Token verification failed", req, res); } if (decoded && _.isObject(decoded)) { if (!decoded?.id) { - errorHandler(401, "User ID is missing from the decoded token.", req, res); + return errorHandler(401, "User ID is missing from the decoded token.", req, res); } (req as any).userID = decoded?.id; const action = (req as any).id; @@ -89,7 +86,7 @@ export default { const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; - errorHandler(403, errorMessage, req, res); + return errorHandler(403, errorMessage, req, res); } next(); } From 0f148a0607d32580eb39e6d35ee380a04ca4760e Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 17:00:59 +0530 Subject: [PATCH 159/311] #OBS-I165: modified error message --- api-service/src/middlewares/RBAC_middleware.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index c00199a5..b8d4032d 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -81,7 +81,7 @@ export default { return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); }); const rolesMessage = rolesWithAccess.length > 0 - ? `Roles with access: ${rolesWithAccess.join(", ")}` + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` : "No roles have this action"; const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; From cd11e2c64cd35ffc55b28ceff709bba1074f99f1 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 15:09:10 +0530 Subject: [PATCH 160/311] #OBS-I227 : fix: threshold of dataset level alerts inside an array. --- command-service/src/command/alert_manager_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 1c989a5c..6d478c3e 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -176,7 +176,7 @@ def create_alert_rule(self, payload: dict) -> ActionResponse: "subComponent": dataset_name, "metric": prom_metric, "operator": operator, - "threshold": threshold, + "threshold": [+threshold], "metricAlias": metric_alias, } }, From 4f8dee1f2cc32b058ee135087873d9f675f5c0cd Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 15:11:51 +0530 Subject: [PATCH 161/311] #OBS-I179 : feat: retire dataset related alerts and metrics on dataset retire --- .../DatasetStatusTransition.ts | 2 +- api-service/src/services/DatasetService.ts | 12 ++++++++++-- api-service/src/services/managers/index.ts | 8 ++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 43d42ba5..4de7a474 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -52,7 +52,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) const userID = (req as any)?.userID; validateDataset(dataset, dataset_id, status); diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index c3284b70..1c864404 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -17,6 +17,7 @@ import { Datasource } from "../models/Datasource"; import { obsrvError } from "../types/ObsrvError"; import { druidHttpService } from "../connections/druidConnection"; import { tableGenerator } from "./TableGenerator"; +import { deleteAlertByDataset, deleteMetricAliasByDataset } from "./managers"; class DatasetService { @@ -258,6 +259,11 @@ class DatasetService { } } + private deleteAlerts = async (dataset: any) => { + await deleteAlertByDataset(dataset); + await deleteMetricAliasByDataset(dataset); + } + retireDataset = async (dataset: Record, updatedBy: any) => { const transaction = await sequelize.transaction(); @@ -267,11 +273,13 @@ class DatasetService { await Datasource.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await DatasetTransformations.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); - await this.deleteDruidSupervisors(dataset); } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } + // Deleting dataset alerts and druid supervisors + await this.deleteDruidSupervisors(dataset); + await this.deleteAlerts(dataset); } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { @@ -379,7 +387,7 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) const connectorsV1 = await getV1Connectors(dataset_id) - const connectors = _.concat(connectorsV1,connectorsV2) + const connectors = _.concat(connectorsV1, connectorsV2) if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index e6cf822f..43e796dc 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -7,7 +7,7 @@ import { Metrics } from "../../models/Metric"; import constants from "./constants"; export const getAlertRule = (id: string) => { - return Alert.findOne({ where: { id } }); + return Alert.findOne({ where: { id }, raw: true }); } const getService = (manager: string) => { @@ -138,11 +138,7 @@ export const deleteAlertByDataset = async (payload: Record) => { export const deleteMetricAliasByDataset = async (payload: Record) => { try { const { name } = payload; - const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) - if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) - for (const payload of metricAliasPayload) { - await payload.destroy() - } + await Metrics.destroy({ where: { component: "datasets", subComponent: name } }) return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; } catch (error: any) { throw new Error(constants.METRIC_ALIAS_NOT_DELETED); From a1e8a07cd2b3d18a88a7ecd1546a22c0fe242c44 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 15:28:15 +0530 Subject: [PATCH 162/311] #OBS-I79: feat: test case fixes --- .../DatasetStatusTransition/DatasetRetire.spec.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 1a0428af..e09dcc41 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -14,6 +14,7 @@ import { Datasource } from "../../../models/Datasource"; import { commandHttpService } from "../../../connections/commandServiceConnection"; import { druidHttpService } from "../../../connections/druidConnection"; import { sequelize } from "../../../connections/databaseConnection"; +import { datasetService } from "../../../services/DatasetService"; chai.use(spies); chai.should(); @@ -46,6 +47,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve([{ datasource_ref: "telemetry" }]) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(druidHttpService, "post", () => { return Promise.resolve({}) }) @@ -91,6 +95,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(commandHttpService, "post", () => { return Promise.resolve({}) }) @@ -130,6 +137,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Datasource, "update", () => { return Promise.resolve({}) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) From 1989e53a366771bf6ac0d787f1b2e60c07e26218 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 18:21:37 +0530 Subject: [PATCH 163/311] #OBS-I79: feat: Alerts get fix --- api-service/src/controllers/Alerts/Alerts.ts | 5 +++-- api-service/src/services/managers/index.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index b3bda19e..76bd9c3a 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -31,8 +31,9 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio const publishAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const { alertId } = req.params; - const rulePayload: Record | null = await getAlertRule(alertId); - if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const ruleModel: Record | null = await getAlertRule(alertId); + if (!ruleModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const rulePayload = ruleModel.toJSON(); const userID = (req as any)?.userID; _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index 43e796dc..585b32b7 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -7,7 +7,7 @@ import { Metrics } from "../../models/Metric"; import constants from "./constants"; export const getAlertRule = (id: string) => { - return Alert.findOne({ where: { id }, raw: true }); + return Alert.findOne({ where: { id } }); } const getService = (manager: string) => { From 2045b1493117bdf4906e51af5f19a92c04bfa0f8 Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:10:41 +0530 Subject: [PATCH 164/311] feat: command tag name change (#252) --- .github/workflows/build_and_deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index db26184e..a51e05bf 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -43,7 +43,7 @@ jobs: context: ./command-service platforms: linux/amd64 push: true - tags: ${{ vars.DOCKERHUB_USERNAME }}/flink-command-service:${{ github.ref_name }} + tags: ${{ vars.DOCKERHUB_USERNAME }}/obsrv-command-service:${{ github.ref_name }} aws-deploy: needs: [build-api-service-image, build-command-service-image] From c59c30908605f13f34a9547e9b1abbbac1f89cca Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:14:40 +0530 Subject: [PATCH 165/311] feat: command tag name change (#252) (#254) --- .github/workflows/build_and_deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index db26184e..a51e05bf 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -43,7 +43,7 @@ jobs: context: ./command-service platforms: linux/amd64 push: true - tags: ${{ vars.DOCKERHUB_USERNAME }}/flink-command-service:${{ github.ref_name }} + tags: ${{ vars.DOCKERHUB_USERNAME }}/obsrv-command-service:${{ github.ref_name }} aws-deploy: needs: [build-api-service-image, build-command-service-image] From ae6ac6bf5bad2f7c3bf5f689c3a6201397a4cc53 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 26 Sep 2024 11:27:50 +0530 Subject: [PATCH 166/311] #0000 fix: ingestion spec cache fix --- api-service/src/services/TableGenerator.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index e22714f6..ec1db785 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -86,6 +86,7 @@ class TableGenerator extends BaseTableGenerator { getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { const { dataset_config, router_config } = dataset + const ingestionSpecDefaults = _.cloneDeep(rawIngestionSpecDefaults) return { "type": "kafka", "spec": { @@ -94,10 +95,10 @@ class TableGenerator extends BaseTableGenerator { "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset), dataset_config.keys_config.partition_key) }, "timestampSpec": { "column": this.getTimestampKey(dataset), "format": "auto" }, "metricsSpec": [], - "granularitySpec": rawIngestionSpecDefaults.granularitySpec + "granularitySpec": ingestionSpecDefaults.granularitySpec }, - "tuningConfig": rawIngestionSpecDefaults.tuningConfig, - "ioConfig": _.merge(rawIngestionSpecDefaults.ioConfig, { + "tuningConfig": ingestionSpecDefaults.tuningConfig, + "ioConfig": _.merge(ingestionSpecDefaults.ioConfig, { "topic": router_config.topic, "inputFormat": { "flattenSpec": { @@ -142,8 +143,8 @@ class TableGenerator extends BaseTableGenerator { } } - private getDruidFlattenSpec = (allFields: Record) => { - const allfields = _.map(allFields, (field) => { + private getDruidFlattenSpec = (fields: Record) => { + const allfields = _.map(fields, (field) => { return { type: "path", expr: field.expr, From 03c2302576dbb6afa74e77ab2aad7f5e675cd032 Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:08:50 +0530 Subject: [PATCH 167/311] Ingestion spec fix (#255) * feat: command tag name change (#252) (#254) * #0000 fix: ingestion spec cache fix --------- Co-authored-by: Ravi Mula --- api-service/src/services/TableGenerator.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index e22714f6..ec1db785 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -86,6 +86,7 @@ class TableGenerator extends BaseTableGenerator { getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { const { dataset_config, router_config } = dataset + const ingestionSpecDefaults = _.cloneDeep(rawIngestionSpecDefaults) return { "type": "kafka", "spec": { @@ -94,10 +95,10 @@ class TableGenerator extends BaseTableGenerator { "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset), dataset_config.keys_config.partition_key) }, "timestampSpec": { "column": this.getTimestampKey(dataset), "format": "auto" }, "metricsSpec": [], - "granularitySpec": rawIngestionSpecDefaults.granularitySpec + "granularitySpec": ingestionSpecDefaults.granularitySpec }, - "tuningConfig": rawIngestionSpecDefaults.tuningConfig, - "ioConfig": _.merge(rawIngestionSpecDefaults.ioConfig, { + "tuningConfig": ingestionSpecDefaults.tuningConfig, + "ioConfig": _.merge(ingestionSpecDefaults.ioConfig, { "topic": router_config.topic, "inputFormat": { "flattenSpec": { @@ -142,8 +143,8 @@ class TableGenerator extends BaseTableGenerator { } } - private getDruidFlattenSpec = (allFields: Record) => { - const allfields = _.map(allFields, (field) => { + private getDruidFlattenSpec = (fields: Record) => { + const allfields = _.map(fields, (field) => { return { type: "path", expr: field.expr, From 1b728ece8b494b90b4a842e853ae898902c7e831 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 30 Sep 2024 18:31:34 +0530 Subject: [PATCH 168/311] #OBS-I230: fix: kafka command --- command-service/requirements.txt | 1 + .../src/command/command_executor.py | 5 ++ command-service/src/command/kafka_command.py | 46 +++++++++++++++++++ command-service/src/config/service_config.yml | 1 + command-service/src/model/data_models.py | 1 + 5 files changed, 54 insertions(+) create mode 100644 command-service/src/command/kafka_command.py diff --git a/command-service/requirements.txt b/command-service/requirements.txt index 6136ac6d..8593fc9f 100644 --- a/command-service/requirements.txt +++ b/command-service/requirements.txt @@ -16,4 +16,5 @@ tenacity==8.2.2 kafka-python-ng==2.2.2 boto3 prometheus-client +confluent-kafka==2.5.3 requests \ No newline at end of file diff --git a/command-service/src/command/command_executor.py b/command-service/src/command/command_executor.py index 665e4636..dffa8155 100644 --- a/command-service/src/command/command_executor.py +++ b/command-service/src/command/command_executor.py @@ -10,6 +10,7 @@ from command.druid_command import DruidCommand from command.flink_command import FlinkCommand from command.telemetry_command import TelemetryCommand +from command.kafka_command import KafkaCommand from config import Config from model.data_models import Action, ActionResponse, CommandPayload from service.db_service import DatabaseService @@ -49,6 +50,9 @@ def __init__(self): self.db_command = DBCommand( db_service=self.db_service, dataset_command=self.dataset_command ) + self.create_kafka_topic = KafkaCommand( + config=self.config_obj, http_service=self.http_service, dataset_command=self.dataset_command + ) self.audit_event_command = TelemetryCommand( telemetry_service=self.telemetry_service, dataset_command=self.dataset_command, @@ -56,6 +60,7 @@ def __init__(self): self.action_commands = {} self.action_commands[Action.START_PIPELINE_JOBS.name] = self.flink_command self.action_commands[Action.MAKE_DATASET_LIVE.name] = self.db_command + self.action_commands[Action.CREATE_KAFKA_TOPIC.name] = self.create_kafka_topic self.action_commands[Action.SUBMIT_INGESTION_TASKS.name] = self.druid_command self.action_commands[Action.DEPLOY_CONNECTORS.name] = self.connector_command self.action_commands[Action.CREATE_ALERT_METRIC.name] = ( diff --git a/command-service/src/command/kafka_command.py b/command-service/src/command/kafka_command.py new file mode 100644 index 00000000..c638f47f --- /dev/null +++ b/command-service/src/command/kafka_command.py @@ -0,0 +1,46 @@ +from command.icommand import ICommand +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.http_service import HttpService +from confluent_kafka.admin import AdminClient, NewTopic +from command.dataset_command import DatasetCommand + +class KafkaCommand(ICommand): + def __init__(self, config: Config, http_service: HttpService, dataset_command: DatasetCommand): + self.config = config + self.http_service = http_service + self.dataset_command = dataset_command + + def execute(self, command_payload: CommandPayload, action: Action): + result = None + if action == Action.CREATE_KAFKA_TOPIC.name: + print( + f"Invoking START_PIPELINE_JOBS command for dataset_id {command_payload.dataset_id}..." + ) + dataset_id = command_payload.dataset_id + live_dataset, data_version = self.dataset_command._check_for_live_record( + dataset_id + ) + topic = live_dataset.router_config['topic'] + result = self.create_kafka_topic(topic, "localhost:9092", 1, 1) + return result + + + def create_kafka_topic(self, topic, broker, num_partitions, replication_factor): + admin_client = AdminClient({'bootstrap.servers': broker}) + print(f"topic is",topic) + new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] + + try: + fs = admin_client.create_topics(new_topic) + for topic, f in fs.items(): + try: + f.result() + print(f"Topic '{topic}' created successfully") + except Exception as e: + print(f"Failed to create topic '{topic}': {e}") + except Exception as e: + print(f"Error creating topic: {e}") + + return ActionResponse(status="OK", status_code=200) \ No newline at end of file diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 8b9f2bdc..6f12a413 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -16,6 +16,7 @@ commands: PUBLISH_DATASET: workflow: - MAKE_DATASET_LIVE + - CREATE_KAFKA_TOPIC - SUBMIT_INGESTION_TASKS - START_PIPELINE_JOBS - DEPLOY_CONNECTORS diff --git a/command-service/src/model/data_models.py b/command-service/src/model/data_models.py index f248e6ba..2ab1611d 100644 --- a/command-service/src/model/data_models.py +++ b/command-service/src/model/data_models.py @@ -14,6 +14,7 @@ class Command(Enum): class Action(Enum): SUBMIT_INGESTION_TASKS = "SUBMIT_INGESTION_TASKS" MAKE_DATASET_LIVE = "MAKE_DATASET_LIVE" + CREATE_KAFKA_TOPIC = "CREATE_KAFKA_TOPIC" MAKE_DATASOURCE_ACTIVE = "MAKE_DATASOURCE_ACTIVE" START_PIPELINE_JOBS = "START_PIPELINE_JOBS" CREATE_ALERT_METRIC = "CREATE_ALERT_METRIC" From 9aea6ba34874dab181101b066c8697287e0d208e Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 30 Sep 2024 20:11:57 +0530 Subject: [PATCH 169/311] #OBS-I230: fix: cache issues changes --- .../DatasetCopy/DatasetCopyHelper.ts | 3 ++- .../DatasetImport/DatasetImportHelper.ts | 3 ++- .../DatasetStatusTransition.ts | 12 +++++---- command-service/Dockerfile | 25 +++++++++---------- command-service/src/command/kafka_command.py | 7 ++++-- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index 3ce8b0fe..f89a9829 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -2,7 +2,8 @@ import * as _ from "lodash"; import { DatasetStatus } from "../../types/DatasetModels"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; import { config } from "../../configs/Config"; -const version = defaultDatasetConfig.version; +const defaultConfigs = _.cloneDeep(defaultDatasetConfig); +const version = defaultConfigs.version; export const updateRecords = (datasetRecord: Record, newDatasetId: string): void => { const dataset_id = newDatasetId; diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 7d06a196..73907183 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -115,7 +115,8 @@ export const migrateExportedDatasetV1 = (requestPayload: Record) => const { drop_duplicates, dedup_key, dedup_period, extraction_key, enabled: is_batch_event } = batch dataset["extraction_config"] = { is_batch_event, extraction_key, dedup_config: { drop_duplicates, dedup_key, dedup_period } } - const { redis_db, redis_db_host, redis_db_port } = defaultDatasetConfig.dataset_config.cache_config; + const defaults = _.cloneDeep(defaultDatasetConfig); + const { redis_db, redis_db_host, redis_db_port } = defaults.dataset_config.cache_config; dataset["dataset_config"] = { indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (type === DatasetType.master) }, keys_config: { data_key, timestamp_key }, diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 4de7a474..5dc3c06f 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -95,8 +95,8 @@ const readyForPublish = async (dataset: Record, updated_by: any) => defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } _.set(draftDataset, "updated_by", updated_by); - _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue ,key) => { - if (key === "created_by"|| key === "updated_by") { + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue, key) => { + if (key === "created_by" || key === "updated_by") { if (objValue !== "SYSTEM") { return objValue; } @@ -182,9 +182,10 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } // 2. Populate redis db for denorm + const defaults = _.cloneDeep(defaultDatasetConfig) draftDataset["denorm_config"] = { - redis_db_host: defaultDatasetConfig.denorm_config.redis_db_host, - redis_db_port: defaultDatasetConfig.denorm_config.redis_db_port, + redis_db_host: defaults.denorm_config.redis_db_host, + redis_db_port: defaults.denorm_config.redis_db_port, denorm_fields: _.map(masterDatasetsStatus, "denorm_field") } } @@ -193,7 +194,8 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) const updateMasterDataConfig = async (draftDataset: Record) => { if (draftDataset.type === "master") { const dataset_config = _.get(draftDataset, "dataset_config") - const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") + const defaultConfigs = _.cloneDeep(defaultDatasetConfig) + const datasetCacheConfig = _.get(defaultConfigs, "dataset_config.cache_config") draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 2ca4edef..2cdfd202 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -1,18 +1,17 @@ -FROM --platform=linux/amd64 python:3.12-alpine -COPY --from=ubuntu /usr/local/bin /usr/local/bin +FROM --platform=linux/amd64 python:3.12-slim -RUN apk update && apk add curl jq vim && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + librdkafka-dev \ + curl \ + jq \ + vim \ + && rm -rf /var/lib/apt/lists/* -RUN cd /tmp && curl -OL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ - && tar -zxvf helm-v3.13.2-linux-amd64.tar.gz \ - && mv linux-amd64/helm /usr/local/bin/helm - -# RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ -# && mv kubectl /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl - -# RUN apk add openssl==3.3.0-r2 libcrypto3==3.3.0-r2 || \ -RUN apk add openssl libcrypto3 -RUN apk upgrade --no-cache +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/local/bin/ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/command-service/src/command/kafka_command.py b/command-service/src/command/kafka_command.py index c638f47f..f27b583c 100644 --- a/command-service/src/command/kafka_command.py +++ b/command-service/src/command/kafka_command.py @@ -16,14 +16,17 @@ def execute(self, command_payload: CommandPayload, action: Action): result = None if action == Action.CREATE_KAFKA_TOPIC.name: print( - f"Invoking START_PIPELINE_JOBS command for dataset_id {command_payload.dataset_id}..." + f"Invoking CREATE_KAFKA_TOPIC command for dataset_id {command_payload.dataset_id}..." ) + self.config_obj = Config() dataset_id = command_payload.dataset_id live_dataset, data_version = self.dataset_command._check_for_live_record( dataset_id ) topic = live_dataset.router_config['topic'] - result = self.create_kafka_topic(topic, "localhost:9092", 1, 1) + brokers = self.config_obj.find("kafka.brokers") + print(f"broker", brokers) + result = self.create_kafka_topic(topic, brokers, 1, 1) return result From a173dbc924b592f0e6cc425f2e49eb96de316dd3 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 1 Oct 2024 15:16:34 +0530 Subject: [PATCH 170/311] #OBS-I230: fix: redis db number change on update --- .../DatasetStatusTransition.ts | 16 ++++++++++++---- .../controllers/DatasetUpdate/DatasetUpdate.ts | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 5dc3c06f..bad6f52e 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -93,6 +93,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); + defaultConfigs = _.omit(defaultConfigs, "dataset_config.cache_config.redis_db"); } _.set(draftDataset, "updated_by", updated_by); _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue, key) => { @@ -193,10 +194,17 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) const updateMasterDataConfig = async (draftDataset: Record) => { if (draftDataset.type === "master") { - const dataset_config = _.get(draftDataset, "dataset_config") - const defaultConfigs = _.cloneDeep(defaultDatasetConfig) - const datasetCacheConfig = _.get(defaultConfigs, "dataset_config.cache_config") - draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } + const datasetConfig = _.get(draftDataset, "dataset_config") + const defaultConfigs = _.cloneDeep(defaultDatasetConfig); + + const defaultCacheConfig = _.get(defaultConfigs, "dataset_config.cache_config"); + const currentCacheConfig = _.get(datasetConfig, "cache_config"); + draftDataset.dataset_config = { + ...datasetConfig, cache_config: { + ...defaultCacheConfig, ..._.omit(currentCacheConfig, "redis_db"), + redis_db: _.get(currentCacheConfig, "redis_db") + } + }; if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() if (_.isEmpty(results)) { diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 7277b1b8..ba426c43 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -67,6 +67,7 @@ const datasetUpdate = async (req: Request, res: Response) => { const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { + const cache_config = _.get(datasetModel, ["dataset_config", "cache_config"]) const dataset: Record = { version_key: Date.now().toString(), name: datasetReq.name || _.get(datasetModel, ["name"]), @@ -76,7 +77,7 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config if(datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config if(datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema - if(datasetReq.dataset_config) dataset["dataset_config"] = datasetReq.dataset_config + if(datasetReq.dataset_config) dataset["dataset_config"] = { ...datasetReq.dataset_config, cache_config } if(datasetReq.transformations_config) dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config) if(datasetReq.denorm_config) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config) From e9f2ebcef6dbd74dcb9a52b45c94de4f22a3b8f0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 1 Oct 2024 15:37:05 +0530 Subject: [PATCH 171/311] #OBS-I187: added model for user --- api-service/src/models/User.ts | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 api-service/src/models/User.ts diff --git a/api-service/src/models/User.ts b/api-service/src/models/User.ts new file mode 100644 index 00000000..aa7c8d37 --- /dev/null +++ b/api-service/src/models/User.ts @@ -0,0 +1,49 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const User = sequelize.define("user", { + id: { + type: DataTypes.STRING, + primaryKey: true + }, + user_name: { + type: DataTypes.STRING + }, + password: { + type: DataTypes.STRING + }, + first_name: { + type: DataTypes.STRING + }, + last_name: { + type: DataTypes.STRING, + }, + provider: { + type: DataTypes.STRING, + }, + email_address: { + type: DataTypes.STRING, + }, + mobile_number: { + type: DataTypes.STRING, + }, + created_on: { + type: DataTypes.STRING, + }, + last_updated_on: { + type: DataTypes.STRING, + }, + roles: { + type: DataTypes.ARRAY(DataTypes.STRING), + defaultValue: ["viewer"] + }, + status: { + type: DataTypes.STRING, + defaultValue: "active" + } +}, { + tableName: "oauth_users", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date" +}) From c8d7a8d8acaa19b88698b684417e381562de6768 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 1 Oct 2024 15:37:59 +0530 Subject: [PATCH 172/311] #OBS-I187: added userService --- api-service/src/services/UserService.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 api-service/src/services/UserService.ts diff --git a/api-service/src/services/UserService.ts b/api-service/src/services/UserService.ts new file mode 100644 index 00000000..6850cb91 --- /dev/null +++ b/api-service/src/services/UserService.ts @@ -0,0 +1,11 @@ +import { User } from "../models/User"; + +class UserService { + + getUser = (where?: Record, attributes?: string[]): Promise => { + return User.findOne({ where, attributes, raw: true }); + } + +} + +export const userService = new UserService(); \ No newline at end of file From b3bb16a58b6459db2803e63e30e9c0b1be38bc00 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 1 Oct 2024 15:38:45 +0530 Subject: [PATCH 173/311] #OBS-I187: modified to verify keycloak token --- .../src/middlewares/RBAC_middleware.ts | 57 +++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index b8d4032d..1aa26e70 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -5,11 +5,12 @@ import { config } from "../configs/Config"; import _ from "lodash"; import userPermissions from "./userPermissions.json"; import httpStatus from "http-status"; +import { userService } from "../services/UserService"; interface AccessControl { - apiGroups : { + apiGroups: { [key: string]: string[]; }, - roles : { + roles: { [key: string]: string[]; } } @@ -42,14 +43,16 @@ const errorHandler = (statusCode: number, message: string, req: Request, res: Re ); }; +const authType = config.authenticationType; + export default { name: "rbac:middleware", - handler: () => (req: Request, res: Response, next: NextFunction) => { + handler: () => async (req: Request, res: Response, next: NextFunction) => { try { if (_.lowerCase(config.is_RBAC_enabled) === "false") { (req as any).userID = "SYSTEM"; next(); - } else { + } else if (authType === "basic") { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { @@ -92,6 +95,52 @@ export default { } }); } + else if (authType === "keycloak") { + const token = req.get("x-user-token"); + + if (!token) { + return res.status(401).json({ message: "No token provided" }); + } + const decoded = jwt.decode(token); + if (!decoded) { + return res.status(401).json({ message: "Invalid token" }); + } + const userCondition: any = {}; + userCondition.id = decoded.sub; + const userDetails = ["roles", "user_name"] + const user = await userService.getUser(userCondition, userDetails); + + if (!user) { + return res.status(404).json({ message: "User not found" }); + } + + (req as any).userID = decoded?.sub; + const action = (req as any).id; + const hasAccess = user?.roles?.some((role: string) => { + const apiGroups = accessControl.roles[role]; + + if (!apiGroups) return false; + + return apiGroups.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); + if (!hasAccess) { + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + if (!apiGroups) return false; + return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + const rolesMessage = rolesWithAccess.length > 0 + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + + return errorHandler(403, errorMessage, req, res); + } + next(); + } } catch (error) { next(error); } From a4e0b048f5c9afe44b12594188419b0be3daa429 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 1 Oct 2024 17:37:20 +0530 Subject: [PATCH 174/311] #OBS-I187: updated rbac middleware --- .../src/middlewares/RBAC_middleware.ts | 183 +++++++++--------- 1 file changed, 95 insertions(+), 88 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 1aa26e70..3cac5ea7 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -43,7 +43,90 @@ const errorHandler = (statusCode: number, message: string, req: Request, res: Re ); }; -const authType = config.authenticationType; + +const basicToken = (token: string, req: Request, res: Response, next: NextFunction) => { + try { + const decoded = jwt.verify(token, config.user_token_public_key); + + if (!decoded || !_.isObject(decoded)) { + return errorHandler(401, "Token verification failed or invalid token", req, res); + } + + (req as any).userID = decoded.id; + const action = (req as any).id; + + if (decoded.roles) { + const hasAccess = decoded.roles.some((role: string) => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); + + if (hasAccess) { + return next(); + }else { + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + + const rolesMessage = rolesWithAccess.length > 0 + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + return errorHandler(403, errorMessage, req, res); + } + } + + return errorHandler(403, "Access denied. User does not have permission to perform this action.", req, res); + } catch (error) { + return errorHandler(401, "Token verification error", req, res); + } +}; + +const keycloakTokenVerify = async (token: string, req: Request, res: Response, next: NextFunction) => { + try { + const decoded = jwt.decode(token); + if (decoded && _.isObject(decoded)) { + (req as any).userID = decoded.sub; + const action = (req as any).id; + const userCondition = { id: decoded.sub }; + const userDetails = ["roles", "user_name"]; + const user = await userService.getUser(userCondition, userDetails); + + if (!user) { + return errorHandler(404, "User not found", req, res); + } + + const hasAccess = user.roles?.some((role: string) => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); + + if(hasAccess){ + return next(); + }else { + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + + const rolesMessage = rolesWithAccess.length > 0 + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + return errorHandler(403, errorMessage, req, res); + } + } + } catch (error) { + return errorHandler(401, "Token decode error", req, res); + } +}; export default { name: "rbac:middleware", @@ -51,98 +134,22 @@ export default { try { if (_.lowerCase(config.is_RBAC_enabled) === "false") { (req as any).userID = "SYSTEM"; - next(); - } else if (authType === "basic") { - const public_key = config.user_token_public_key; - const token = req.get("x-user-token"); - if (!token) { - return errorHandler(401, "No token provided", req, res); - } - jwt.verify(token as string, public_key, (err, decoded) => { - if (err) { - return errorHandler(401, "Token verification failed", req, res); - } - if (decoded && _.isObject(decoded)) { - if (!decoded?.id) { - return errorHandler(401, "User ID is missing from the decoded token.", req, res); - } - (req as any).userID = decoded?.id; - const action = (req as any).id; - const hasAccess = decoded?.roles?.some((role: string) => { - const apiGroups = accessControl.roles[role]; - - if (!apiGroups) return false; - - return apiGroups.some((apiGroup: string) => - accessControl.apiGroups[apiGroup]?.includes(action) - ); - }); - if (!hasAccess) { - const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { - const apiGroups = accessControl.roles[role]; - if (!apiGroups) return false; - return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); - }); - const rolesMessage = rolesWithAccess.length > 0 - ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` - : "No roles have this action"; - - const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; - - return errorHandler(403, errorMessage, req, res); - } - next(); - } - }); + return next(); } - else if (authType === "keycloak") { - const token = req.get("x-user-token"); - - if (!token) { - return res.status(401).json({ message: "No token provided" }); - } - const decoded = jwt.decode(token); - if (!decoded) { - return res.status(401).json({ message: "Invalid token" }); - } - const userCondition: any = {}; - userCondition.id = decoded.sub; - const userDetails = ["roles", "user_name"] - const user = await userService.getUser(userCondition, userDetails); - - if (!user) { - return res.status(404).json({ message: "User not found" }); - } - - (req as any).userID = decoded?.sub; - const action = (req as any).id; - const hasAccess = user?.roles?.some((role: string) => { - const apiGroups = accessControl.roles[role]; - if (!apiGroups) return false; + const token = req.get("x-user-token"); + if (!token) { + return errorHandler(401, "No token provided", req, res); + } - return apiGroups.some((apiGroup: string) => - accessControl.apiGroups[apiGroup]?.includes(action) - ); - }); - if (!hasAccess) { - const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { - const apiGroups = accessControl.roles[role]; - if (!apiGroups) return false; - return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); - }); - const rolesMessage = rolesWithAccess.length > 0 - ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` - : "No roles have this action"; - - const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; - - return errorHandler(403, errorMessage, req, res); - } - next(); + const decoded = jwt.decode(token); + if (decoded && _.isObject(decoded) && decoded.roles) { + return basicToken(token, req, res, next); + } else { + return await keycloakTokenVerify(token, req, res, next); } } catch (error) { - next(error); + return next(error); } }, }; From 9e33752165434309446b3fff75437defe6736132 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 2 Oct 2024 18:23:37 +0530 Subject: [PATCH 175/311] #OBS-I187: add seperate function to check access --- .../src/middlewares/RBAC_middleware.ts | 81 ++++++++----------- 1 file changed, 35 insertions(+), 46 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 3cac5ea7..10fda3e2 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -44,10 +44,41 @@ const errorHandler = (statusCode: number, message: string, req: Request, res: Re }; +const checkAccess = (decoded: any, action: string, req: Request, res: Response) => { + if (decoded.roles) { + const hasAccess = decoded.roles.some((role: string) => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); + + if (hasAccess) { + return true; + } else { + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + return apiGroups?.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + + const rolesMessage = rolesWithAccess.length > 0 + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + errorHandler(403, errorMessage, req, res); + return false; + } + } + + errorHandler(403, "Access denied. User does not have permission to perform this action.", req, res); + return false; +}; + const basicToken = (token: string, req: Request, res: Response, next: NextFunction) => { try { const decoded = jwt.verify(token, config.user_token_public_key); - + if (!decoded || !_.isObject(decoded)) { return errorHandler(401, "Token verification failed or invalid token", req, res); } @@ -55,32 +86,9 @@ const basicToken = (token: string, req: Request, res: Response, next: NextFuncti (req as any).userID = decoded.id; const action = (req as any).id; - if (decoded.roles) { - const hasAccess = decoded.roles.some((role: string) => { - const apiGroups = accessControl.roles[role]; - return apiGroups?.some((apiGroup: string) => - accessControl.apiGroups[apiGroup]?.includes(action) - ); - }); - - if (hasAccess) { - return next(); - }else { - const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { - const apiGroups = accessControl.roles[role]; - return apiGroups?.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); - }); - - const rolesMessage = rolesWithAccess.length > 0 - ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` - : "No roles have this action"; - - const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; - return errorHandler(403, errorMessage, req, res); - } + if (checkAccess(decoded, action, req, res)) { + return next(); } - - return errorHandler(403, "Access denied. User does not have permission to perform this action.", req, res); } catch (error) { return errorHandler(401, "Token verification error", req, res); } @@ -100,27 +108,8 @@ const keycloakTokenVerify = async (token: string, req: Request, res: Response, n return errorHandler(404, "User not found", req, res); } - const hasAccess = user.roles?.some((role: string) => { - const apiGroups = accessControl.roles[role]; - return apiGroups?.some((apiGroup: string) => - accessControl.apiGroups[apiGroup]?.includes(action) - ); - }); - - if(hasAccess){ + if (checkAccess(user, action, req, res)) { return next(); - }else { - const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { - const apiGroups = accessControl.roles[role]; - return apiGroups?.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); - }); - - const rolesMessage = rolesWithAccess.length > 0 - ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` - : "No roles have this action"; - - const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; - return errorHandler(403, errorMessage, req, res); } } } catch (error) { From 2faf429eb2d7c6e43999699ecd3e06f11b47b83c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 2 Oct 2024 20:32:59 +0530 Subject: [PATCH 176/311] #OBS-I230: fix: integration fixes --- command-service/Dockerfile | 2 ++ command-service/src/command/alert_manager_command.py | 10 ++++++++-- command-service/src/command/db_command.py | 7 ++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 2cdfd202..4e7fe11a 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -12,6 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ chmod +x kubectl && \ mv kubectl /usr/local/bin/ + +RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 6d478c3e..3b6a82b9 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -142,7 +142,10 @@ def create_alert_metric( ) else: error_data = json.loads(response.body) - error_message = error_data["params"]["errmsg"] + if "params" in error_data and "errmsg" in error_data["params"]: + error_message = error_data["params"]["errmsg"] + else: + error_message = "Failed to publish alerts" return ActionResponse( status="ERROR", status_code=500, @@ -195,7 +198,10 @@ def create_alert_rule(self, payload: dict) -> ActionResponse: self.publish_alert_rule(alert_id=alert_id) else: error_data = json.loads(response.body) - error_message = error_data["params"]["errmsg"] + if "params" in error_data and "errmsg" in error_data["params"]: + error_message = error_data["params"]["errmsg"] + else: + error_message = "Failed to publish alerts" return ActionResponse( status="ERROR", status_code=500, diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 20e9c527..5a1c6b86 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -257,13 +257,14 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): data_class = DatasetConnectorConfigDraft, data = record ) current_timestamp = dt.now() + operations_config = connector_config.operations_config if connector_config.operations_config is not None else {} if connector_config.version == 'v2': params = ( connector_config.id, dataset_id, connector_config.connector_id, - json.dumps(connector_config.connector_config).replace("'", "''"), - json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.connector_config, + json.dumps(operations_config).replace("'", "''"), connector_config.data_format, DatasetStatusType.Live.name, json.dumps(emptyJson), @@ -274,7 +275,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): current_timestamp, current_timestamp, - json.dumps(connector_config.connector_config).replace("'", "''"), + connector_config.connector_config, json.dumps(connector_config.operations_config).replace("'", "''"), connector_config.data_format, draft_dataset_record.get('updated_by'), From 811ce895eecd54c6f7bd9c792a7b21b7c4e35091 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 3 Oct 2024 13:27:58 +0530 Subject: [PATCH 177/311] #OBS-I230: fix: kafka command fixes --- command-service/src/command/kafka_command.py | 50 ++++++++++--------- command-service/src/config/service_config.yml | 4 +- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/command-service/src/command/kafka_command.py b/command-service/src/command/kafka_command.py index f27b583c..13691e1f 100644 --- a/command-service/src/command/kafka_command.py +++ b/command-service/src/command/kafka_command.py @@ -3,7 +3,7 @@ from config import Config from model.data_models import Action, ActionResponse, CommandPayload from service.http_service import HttpService -from confluent_kafka.admin import AdminClient, NewTopic +from confluent_kafka.admin import AdminClient, NewTopic, KafkaError from command.dataset_command import DatasetCommand class KafkaCommand(ICommand): @@ -14,36 +14,38 @@ def __init__(self, config: Config, http_service: HttpService, dataset_command: D def execute(self, command_payload: CommandPayload, action: Action): result = None - if action == Action.CREATE_KAFKA_TOPIC.name: - print( - f"Invoking CREATE_KAFKA_TOPIC command for dataset_id {command_payload.dataset_id}..." - ) - self.config_obj = Config() - dataset_id = command_payload.dataset_id - live_dataset, data_version = self.dataset_command._check_for_live_record( - dataset_id - ) - topic = live_dataset.router_config['topic'] - brokers = self.config_obj.find("kafka.brokers") - print(f"broker", brokers) - result = self.create_kafka_topic(topic, brokers, 1, 1) - return result + dataset_id = command_payload.dataset_id + live_dataset, data_version = self.dataset_command._check_for_live_record(dataset_id) + if live_dataset is None: + if action == Action.CREATE_KAFKA_TOPIC.name: + print( + f"Invoking CREATE_KAFKA_TOPIC command for dataset_id {dataset_id}..." + ) + draft_dataset_record = self.dataset_command._get_draft_dataset(dataset_id) + topic = draft_dataset_record.get("router_config")["topic"] + broker = self.config.find("kafka.brokers") + num_partitions = self.config.find("kafka.no_of_partitions") + replication_factor = self.config.find("kafka.replication_factor") + print(f"topic is", topic) + result = self.create_kafka_topic(topic, broker, num_partitions, replication_factor) + return result + return ActionResponse(status="OK", status_code=200) def create_kafka_topic(self, topic, broker, num_partitions, replication_factor): admin_client = AdminClient({'bootstrap.servers': broker}) - print(f"topic is",topic) new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] + errValue = ActionResponse(status="ERROR", status_code=500) try: fs = admin_client.create_topics(new_topic) for topic, f in fs.items(): - try: - f.result() - print(f"Topic '{topic}' created successfully") - except Exception as e: - print(f"Failed to create topic '{topic}': {e}") + f.result() + print(f"Topic '{topic}' created successfully") + return ActionResponse(status="OK", status_code=200) + except KafkaError as topicEx: + print(f"Error:", topicEx) + return errValue except Exception as e: - print(f"Error creating topic: {e}") - - return ActionResponse(status="OK", status_code=200) \ No newline at end of file + print(f"Error creating topic: {e}") + return errValue \ No newline at end of file diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 6f12a413..80276238 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -15,8 +15,8 @@ flink: commands: PUBLISH_DATASET: workflow: - - MAKE_DATASET_LIVE - CREATE_KAFKA_TOPIC + - MAKE_DATASET_LIVE - SUBMIT_INGESTION_TASKS - START_PIPELINE_JOBS - DEPLOY_CONNECTORS @@ -167,6 +167,8 @@ kafka: brokers: localhost:9092 telemetry: topic: system.telemetry.events + replication_factor: 1 + no_of_partitions: 1 connector_jobs: spark: From 8b0ec90cfc43b1074d0296eed14ce536dcdbb321 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 4 Oct 2024 15:13:59 +0530 Subject: [PATCH 178/311] #OBS-I247: fix: feedback changes. --- command-service/src/command/kafka_command.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/command-service/src/command/kafka_command.py b/command-service/src/command/kafka_command.py index 13691e1f..60578d5c 100644 --- a/command-service/src/command/kafka_command.py +++ b/command-service/src/command/kafka_command.py @@ -3,7 +3,7 @@ from config import Config from model.data_models import Action, ActionResponse, CommandPayload from service.http_service import HttpService -from confluent_kafka.admin import AdminClient, NewTopic, KafkaError +from confluent_kafka.admin import AdminClient, NewTopic, KafkaError, KafkaException from command.dataset_command import DatasetCommand class KafkaCommand(ICommand): @@ -33,18 +33,21 @@ def execute(self, command_payload: CommandPayload, action: Action): def create_kafka_topic(self, topic, broker, num_partitions, replication_factor): - admin_client = AdminClient({'bootstrap.servers': broker}) - new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] errValue = ActionResponse(status="ERROR", status_code=500) try: + admin_client = AdminClient({'bootstrap.servers': broker}) + new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] fs = admin_client.create_topics(new_topic) for topic, f in fs.items(): f.result() print(f"Topic '{topic}' created successfully") return ActionResponse(status="OK", status_code=200) - except KafkaError as topicEx: - print(f"Error:", topicEx) + except (KafkaError, KafkaException) as kafkaErr: + print(f"Kafka exception:", kafkaErr) + return errValue + except RuntimeError as e: + print(f"Runtime exception: {e}") return errValue except Exception as e: print(f"Error creating topic: {e}") From 252982a14b300d3e77c1be4b637cffda525aa949 Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Fri, 4 Oct 2024 15:40:18 +0530 Subject: [PATCH 179/311] #OBS-I230: fix: kafka command fixes (#258) * #OBS-I230: fix: kafka command fixes * #OBS-I247: fix: feedback changes. --------- Co-authored-by: Ravi Mula --- command-service/src/command/kafka_command.py | 57 ++++++++++--------- command-service/src/config/service_config.yml | 4 +- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/command-service/src/command/kafka_command.py b/command-service/src/command/kafka_command.py index f27b583c..60578d5c 100644 --- a/command-service/src/command/kafka_command.py +++ b/command-service/src/command/kafka_command.py @@ -3,7 +3,7 @@ from config import Config from model.data_models import Action, ActionResponse, CommandPayload from service.http_service import HttpService -from confluent_kafka.admin import AdminClient, NewTopic +from confluent_kafka.admin import AdminClient, NewTopic, KafkaError, KafkaException from command.dataset_command import DatasetCommand class KafkaCommand(ICommand): @@ -14,36 +14,41 @@ def __init__(self, config: Config, http_service: HttpService, dataset_command: D def execute(self, command_payload: CommandPayload, action: Action): result = None - if action == Action.CREATE_KAFKA_TOPIC.name: - print( - f"Invoking CREATE_KAFKA_TOPIC command for dataset_id {command_payload.dataset_id}..." - ) - self.config_obj = Config() - dataset_id = command_payload.dataset_id - live_dataset, data_version = self.dataset_command._check_for_live_record( - dataset_id - ) - topic = live_dataset.router_config['topic'] - brokers = self.config_obj.find("kafka.brokers") - print(f"broker", brokers) - result = self.create_kafka_topic(topic, brokers, 1, 1) - return result + dataset_id = command_payload.dataset_id + live_dataset, data_version = self.dataset_command._check_for_live_record(dataset_id) + if live_dataset is None: + if action == Action.CREATE_KAFKA_TOPIC.name: + print( + f"Invoking CREATE_KAFKA_TOPIC command for dataset_id {dataset_id}..." + ) + draft_dataset_record = self.dataset_command._get_draft_dataset(dataset_id) + topic = draft_dataset_record.get("router_config")["topic"] + broker = self.config.find("kafka.brokers") + num_partitions = self.config.find("kafka.no_of_partitions") + replication_factor = self.config.find("kafka.replication_factor") + print(f"topic is", topic) + result = self.create_kafka_topic(topic, broker, num_partitions, replication_factor) + return result + return ActionResponse(status="OK", status_code=200) def create_kafka_topic(self, topic, broker, num_partitions, replication_factor): - admin_client = AdminClient({'bootstrap.servers': broker}) - print(f"topic is",topic) - new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] + errValue = ActionResponse(status="ERROR", status_code=500) try: + admin_client = AdminClient({'bootstrap.servers': broker}) + new_topic = [NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor)] fs = admin_client.create_topics(new_topic) for topic, f in fs.items(): - try: - f.result() - print(f"Topic '{topic}' created successfully") - except Exception as e: - print(f"Failed to create topic '{topic}': {e}") + f.result() + print(f"Topic '{topic}' created successfully") + return ActionResponse(status="OK", status_code=200) + except (KafkaError, KafkaException) as kafkaErr: + print(f"Kafka exception:", kafkaErr) + return errValue + except RuntimeError as e: + print(f"Runtime exception: {e}") + return errValue except Exception as e: - print(f"Error creating topic: {e}") - - return ActionResponse(status="OK", status_code=200) \ No newline at end of file + print(f"Error creating topic: {e}") + return errValue \ No newline at end of file diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 6f12a413..80276238 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -15,8 +15,8 @@ flink: commands: PUBLISH_DATASET: workflow: - - MAKE_DATASET_LIVE - CREATE_KAFKA_TOPIC + - MAKE_DATASET_LIVE - SUBMIT_INGESTION_TASKS - START_PIPELINE_JOBS - DEPLOY_CONNECTORS @@ -167,6 +167,8 @@ kafka: brokers: localhost:9092 telemetry: topic: system.telemetry.events + replication_factor: 1 + no_of_partitions: 1 connector_jobs: spark: From 248b0591f44366d30dd7f26edc5d0a455f62f31f Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Fri, 18 Oct 2024 15:18:09 +0530 Subject: [PATCH 180/311] #OBS-I218 update kafka connector image tags --- .../helm-charts/flink-connector/values.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index dd08dafa..573952f1 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -243,11 +243,14 @@ serviceMonitor: port: prom flink_jobs: - "kafka-connector-1-0-0": - "enabled": "false" - "connector_id": "kafka-connector-1.0.0" - "source": "kafka-connector-1.0.0" - "main_program": "kafka-connector-1.0.0.jar" + kafka-connector-1-0-0: + registry: sanketikahub + repository: flink-connectors + tag: 1.1.0 + enabled: "false" + connector_id: "kafka-connector-1.0.0" + source: "kafka-connector-1.0.0" + main_program: "kafka-connector-1.0.0.jar" commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file From e84dc4b7615214ad64412a5f3947215ac604772d Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 21 Oct 2024 11:02:25 +0530 Subject: [PATCH 181/311] #OBS-I250: Generation of AUDIT events in the openTelemetry format --- api-service/src/otel/OTelService.ts | 163 ++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 api-service/src/otel/OTelService.ts diff --git a/api-service/src/otel/OTelService.ts b/api-service/src/otel/OTelService.ts new file mode 100644 index 00000000..087381e6 --- /dev/null +++ b/api-service/src/otel/OTelService.ts @@ -0,0 +1,163 @@ +import { Counter, diag, DiagConsoleLogger, DiagLogLevel, Meter, metrics } from '@opentelemetry/api'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { LoggerProvider, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import logger from '../logger'; +import * as logsAPI from '@opentelemetry/api-logs'; + + +export class OTelService { + private static meterProvider: MeterProvider; + private static loggerProvider: LoggerProvider; + private static tracerProvider: NodeTracerProvider; + + public static init() { + const collectorEndpoint = process.env.OTEL_COLLECTOR_ENDPOINT || 'http://localhost:4318'; + this.tracerProvider = this.createTracerProvider(collectorEndpoint); // Store the tracer provider + this.meterProvider = this.createMeterProvider(collectorEndpoint); // Store the meter provider + this.loggerProvider = this.createLoggerProvider(collectorEndpoint); // Store the logger provider + + // Register the global tracer, meter, and logger providers + this.tracerProvider.register(); + this.setGlobalMeterProvider(this.meterProvider); + + logger.info("OpenTelemetry Service Initialized"); + + // Add shutdown hook + process.on('SIGTERM', async () => { + await this.tracerProvider.shutdown(); + await this.meterProvider.shutdown(); + await this.loggerProvider.shutdown(); + }); + } + + private static createTracerProvider(endpoint: string) { + const traceExporter = new OTLPTraceExporter({ + url: `${endpoint}/v1/traces`, + }); + + const tracerProvider = new NodeTracerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + tracerProvider.addSpanProcessor(new BatchSpanProcessor(traceExporter)); + + return tracerProvider; + } + + private static createMeterProvider(endpoint: string) { + const metricExporter = new OTLPMetricExporter({ + url: `${endpoint}/v1/metrics`, + }); + + const meterProvider = new MeterProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + meterProvider.addMetricReader( + new PeriodicExportingMetricReader({ + exporter: metricExporter, + exportIntervalMillis: 10000, + }) + ); + + return meterProvider; + } + + private static createLoggerProvider(endpoint: string) { + const logExporter = new OTLPLogExporter({ + url: `${endpoint}/v1/logs`, + }); + + const loggerProvider = new LoggerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + loggerProvider.addLogRecordProcessor( + new BatchLogRecordProcessor(logExporter) + ); + + return loggerProvider; + } + + // Helper method to create a Resource with service name + private static createServiceResource(serviceName: string) { + return new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: serviceName, + }); + } + + private static setGlobalMeterProvider(meterProvider: MeterProvider) { + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); + diag.info('Registering MeterProvider globally.'); + metrics.setGlobalMeterProvider(meterProvider); + } + + // Method to create a counter metric + public static createCounterMetric(name: string): Counter { + const meter = this.getMeterProvider(); // Use the updated getMeterProvider method + const counter = meter.createCounter(name, { + description: 'Counts the number of API calls', + }); + return counter; + } + + public static getMeterProvider(): Meter { + return this.meterProvider.getMeter('obsrv-api-service'); + } + + public static getLoggerProvider(): LoggerProvider { + return this.loggerProvider; + } + + public static getTracerProvider(): NodeTracerProvider { + return this.tracerProvider; + } + + // Method to record the counter metric + public static recordCounter(counter: Counter, value: number) { + counter.add(value, { + // Optional attributes can be added here + service: 'obsrv-api-service', + }); + } + + // Method to log messages + public static log() { + const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); // Retrieve a logger instance + + loggerInstance.emit({ + severityNumber: logsAPI.SeverityNumber.INFO, + severityText: 'INFO', + body: 'test', + attributes: { 'log.type': 'LogRecord' }, + }); + } + + public static emitAuditLog(auditLog: Record) { + const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); + + // Construct the log record + const logRecord = { + severityNumber: logsAPI.SeverityNumber.INFO, // or ERROR depending on the context + severityText: 'INFO', + body: JSON.stringify(auditLog), // Convert the log object to a string + attributes: { + 'log.type': 'AuditLog', + ...auditLog, // Include the whole log object as attributes if necessary + }, + }; + + // Emit the log record to OpenTelemetry + loggerInstance.emit(logRecord); + + // Log the same message to Winston (optional) + logger.info("Audit log emitted", { auditLog }); + } +} From 3c4901f07ba86b756543d78408c4a256fa46012f Mon Sep 17 00:00:00 2001 From: SurabhiAngadi <138881390+SurabhiAngadi@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:22:06 +0530 Subject: [PATCH 182/311] #OBS-I288: fix: removed data_format field from connector_instance (#260) --- command-service/src/command/db_command.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 5a1c6b86..09003482 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -265,7 +265,6 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): connector_config.connector_id, connector_config.connector_config, json.dumps(operations_config).replace("'", "''"), - connector_config.data_format, DatasetStatusType.Live.name, json.dumps(emptyJson), json.dumps(emptyJson), @@ -277,7 +276,6 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): connector_config.connector_config, json.dumps(connector_config.operations_config).replace("'", "''"), - connector_config.data_format, draft_dataset_record.get('updated_by'), current_timestamp, current_timestamp, @@ -285,7 +283,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): ) insert_query = f""" INSERT INTO connector_instances(id, dataset_id, connector_id, connector_config, operations_config, - data_format, status, connector_state, connector_stats, created_by, updated_by, created_date, + status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES ( %s, @@ -300,13 +298,11 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): %s, %s, %s, - %s, %s ) ON CONFLICT (id) DO UPDATE SET connector_config = %s, operations_config = %s, - data_format = %s, updated_by = %s, updated_date = %s, published_date = %s, From b9e496b27c21fd0a58dfe1415768d85980633d42 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 24 Oct 2024 16:29:46 +0530 Subject: [PATCH 183/311] #OBS-I285: query api fix to validate and set limit to sql queries --- .../src/controllers/DataOut/QueryValidator.ts | 20 ++++++++----------- api-service/src/helpers/ResponseHandler.ts | 6 +++--- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 4c47a90e..d0478474 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -7,9 +7,7 @@ import logger from "../../logger"; import { druidHttpService, getDatasourceListFromDruid } from "../../connections/druidConnection"; import { apiId } from "./DataOutController"; import { ErrorObject } from "../../types/ResponseModel"; -import { Parser } from "node-sql-parser"; import { obsrvError } from "../../types/ObsrvError"; -const parser = new Parser(); const momentFormat = "YYYY-MM-DD HH:MM:SS"; let dataset_id: string; @@ -79,17 +77,15 @@ const setQueryLimits = (queryPayload: any) => { } if (_.isString(queryPayload?.query)) { - const vocabulary: any = parser.astify(queryPayload?.query); - const isLimitIncludes = JSON.stringify(vocabulary); - if (_.includes(isLimitIncludes, "{{LIMIT}}")) { - return queryPayload?.query + let query = queryPayload.query; + if (/\blimit\b/i.test(query)) { + return query; } - const limit = _.get(vocabulary, "limit") - if (limit === null) { - _.set(vocabulary, "limit.value[0].value", queryRules.common.maxResultRowLimit) - _.set(vocabulary, "limit.value[0].type", "number") - const convertToSQL = parser.sqlify(vocabulary); - queryPayload.query = convertToSQL + const limitRegex = /\bLIMIT\b\s+\d+/i; + if (!limitRegex.test(query)) { + const maxLimit = _.get(queryRules, "common.maxResultRowLimit"); + query += ` LIMIT ${maxLimit}`; + queryPayload.query = query; } } } diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 99aee816..3c1273e2 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -15,8 +15,8 @@ const ResponseHandler = { entity && onSuccess(req, res) }, - routeNotFound: (req: Request, res: Response, next: NextFunction) => { - next({ statusCode: httpStatus.NOT_FOUND, message: httpStatus["404"], errCode: httpStatus["404_NAME"] }); + routeNotFound: (req: Request, res: Response) => { + ResponseHandler.obsrvErrorResponse({ statusCode: httpStatus.NOT_FOUND, message: "Route not found", errCode: httpStatus["404_NAME"], code: "ROUTE_NOT_FOUND", err: undefined, data: "", datasetId: "" }, req, res) }, refactorResponse: ({ id = "api", ver = "v2", params = { status: "SUCCESS" }, responseCode = httpStatus["200_NAME"], result = {}, msgid = "", resmsgid = "" }): IResponse => { @@ -42,7 +42,7 @@ const ResponseHandler = { const resmsgid = _.get(res, "resmsgid") const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); - entity && onObsrvFailure(req,res,error) + entity && onObsrvFailure(req, res, error) }, setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { From 5048efe8c50a612899bf95e04ece7b4faafce7b6 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Thu, 24 Oct 2024 16:54:51 +0530 Subject: [PATCH 184/311] #I285: removed empty objects and arrays from sample data (#261) * #I285: removed empty objects and arrays from sample data * Fix code scanning alert no. 98: Loop bound injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * #I285: lint issues fixed --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../GenerateDataSchema/GenerateDataSchema.ts | 5 +- .../DataSchemaService.ts | 60 ++++++++++++++----- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts index a37e5d13..11bcee7e 100644 --- a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -32,7 +32,7 @@ const dataSchema = async (req: Request, res: Response) => { const request = req.body.request const dataSchemaSpec = schemaGenerate(request.data, request.config) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataSchemaSpec }); - + } const schemaGenerate = (sample: Map[], config: Record): any => { @@ -46,13 +46,14 @@ const schemaGenerate = (sample: Map[], config: Record) result.schema = removeFormats(result.schema) return result } else { - let schema = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + let { schema, removedKeys } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); schema = schemaArrayValidator.validate(schema) const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) rollupInfo = schemaCardinalityAnalyser.analyse() const result = process(schema, dataset) result.schema = removeNonIndexColumns(result.schema) result.schema = removeFormats(result.schema) + !_.isEmpty(removedKeys) && _.set(result, "removedKeys", removedKeys) return result } } diff --git a/api-service/src/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts index 67bde9e7..aec2f8d3 100644 --- a/api-service/src/services/SchemaGenerateService/DataSchemaService.ts +++ b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts @@ -5,17 +5,17 @@ import moment from "moment"; import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException"; const DATE_FORMATS = [ - "MM/DD/YYYY","DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", + "MM/DD/YYYY", "DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", "DD-MM-YYYY", "MM-DD-YYYY", "MM-DD-YYYY HH:mm:ss", "YYYY/MM/DD HH:mm:ss", - "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", - "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", - "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", - "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", + "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", + "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", + "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", "YYYY-DD-MMTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ss.SSSZ", "DD/MM/YYYYTHH:mm:ss.SSSZ", "YYYY-DD-MMTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss.SSS", "DD/MM/YYYYTHH:mm:ss.SSS", "MM-DD-YYYYTHH:mm:ss.SSSZ", "DD-MM-YYYYTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ssZ", - "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", - "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", + "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", + "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", "YYYY/MM/DDTHH:mm:ss", "DD/MM/YYYYTHH:mm:ss", "DD-MM-YYYY HH:mm:ss.SSSZ", "YYYY-MM-DD HH:mm:ss.SSSZ", "YYYY-DD-MM HH:mm:ss.SSSZ", "YYYY/MM/DD HH:mm:ss.SSSZ", "DD/MM/YYYY HH:mm:ss.SSSZ", "MM-DD-YYYY HH:mm:ss.SSSZ", "DD-MM-YYYY HH:mm:ssZ", "YYYY-MM-DD HH:mm:ssZ", "YYYY-DD-MM HH:mm:ssZ", @@ -38,16 +38,30 @@ const DATE_FORMATS = [ export class SchemaInference { public inferSchema(sample: any) { - const schema = _.map(sample, (value): any => this.validateEpoch(inferSchema(value).toJSONSchema({ includeSchema: true }), value, "properties")) - return schema + if (!Array.isArray(sample)) { + throw new SchemaGenerationException("Invalid input: sample must be an array.", httpStatus.BAD_REQUEST); + } + const removedAllKeys: any[] = [] + const schema = _.map(sample, (value): any => { + const { cleanedData, removedKeys } = this.removeEmpty(value) + removedAllKeys.push(...removedKeys) + return this.validateEpoch(inferSchema(cleanedData).toJSONSchema({ includeSchema: true }), cleanedData, "properties") + }) + return { schema, removedKeys: removedAllKeys } } public inferBatchSchema(sample: Map[], extractionKey: string) { - return _.flatMap(sample, (value) => { + if (!Array.isArray(sample)) { + throw new SchemaGenerationException("Invalid input: sample must be an array.", httpStatus.BAD_REQUEST); + } + const removedAllKeys: any[] = [] + const schema = _.flatMap(sample, (value) => { if (extractionKey) { const extracted = _.get(value, extractionKey); if (extracted) { - return this.inferSchema(extracted); + const { schema, removedKeys } = this.inferSchema(extracted); + removedAllKeys.push(...removedKeys) + return schema } else { throw new SchemaGenerationException("Unable to extract the batch data.", httpStatus.BAD_REQUEST); } @@ -55,6 +69,22 @@ export class SchemaInference { throw new SchemaGenerationException("Extraction key not found.", httpStatus.BAD_REQUEST); } }) + return { schema, removedKeys: removedAllKeys } + } + // Only removes empty object and array at all the levels + private removeEmpty(data: any, parentKey = "", removedKeys: any[] = []) { + Object.keys(data).forEach((key) => { + const value = data[key]; + const fullKey = parentKey ? `${parentKey}.${key}` : key; + if (typeof value === "object" && value !== null) { + this.removeEmpty(value, fullKey, removedKeys); + if (_.isEmpty(value)) { + delete data[key]; + removedKeys.push({ "key": fullKey, value }); + } + } + }); + return { cleanedData: data, removedKeys }; } private validateEpoch(schema: any, sample: any, path: any) { @@ -67,7 +97,7 @@ export class SchemaInference { if (isValidTimestamp) { _.set(schema, `${path}.${key}.format`, type) } - else if(format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) { + else if (format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) { _.unset(schema, `${path}.${key}.format`) } }); @@ -79,7 +109,7 @@ export class SchemaInference { const epochRegex = /^\d+$/ig; switch (dataType) { case "string": - if(epochRegex.test(value)){ + if (epochRegex.test(value)) { const parsedValue = parseInt(value, 10); // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds return { @@ -93,12 +123,12 @@ export class SchemaInference { case "number": // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds return { - isValidTimestamp: value >= 946684800 && moment(value).isValid(), + isValidTimestamp: value >= 946684800 && moment(value).isValid(), type: "epoch" }; default: return { - isValidTimestamp: false, + isValidTimestamp: false, type: "" }; } From a072199ebb5f66828e364f68bb154d934eb566c5 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 24 Oct 2024 17:44:20 +0530 Subject: [PATCH 185/311] #OBS-I285: query api to parse sql query first then do regex check with postman --- ...srv API Service.postman_collection_v2.json | 1882 --------- ... => Obsrv v2 apis.postman_collection.json} | 3627 ++++++++++++++--- .../src/controllers/DataOut/QueryValidator.ts | 51 +- 3 files changed, 3151 insertions(+), 2409 deletions(-) delete mode 100644 api-service/postman-collection/Obsrv API Service.postman_collection_v2.json rename api-service/postman-collection/{updated_v2_collection.json => Obsrv v2 apis.postman_collection.json} (67%) diff --git a/api-service/postman-collection/Obsrv API Service.postman_collection_v2.json b/api-service/postman-collection/Obsrv API Service.postman_collection_v2.json deleted file mode 100644 index a05836e4..00000000 --- a/api-service/postman-collection/Obsrv API Service.postman_collection_v2.json +++ /dev/null @@ -1,1882 +0,0 @@ -{ - "info": { - "_postman_id": "01a69af1-9cf2-4d45-8b11-5fb6e06ca3ef", - "name": "Obsrv API Service", - "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", - "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "26192103", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-01a69af1-9cf2-4d45-8b11-5fb6e06ca3ef?action=share&source=collection_link&creator=26192103" - }, - "item": [ - { - "name": "Dataset CRUD APIs", - "item": [ - { - "name": "Dataset create", - "protocolProfileBehavior": { - "disabledSystemHeaders": {} - }, - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n \"tag1\",\n \"tag2\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create", - "description": "This API allows you to create new datasets used by the analytical data source." - }, - "response": [ - { - "name": "Dataset Success", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n \"tag1\",\n \"tag2\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:56:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\":\"78hj80j9-d61e-4d4f-be78-181834eeff6d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"sb-telemetry.1\",\n \"version_key\": \"1713442037275\"\n }\n}" - }, - { - "name": "Master Dataset Success", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry-master\",\n \"type\": \"master-dataset\",\n \"name\": \"sb-telemetry-master\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": []\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:56:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\":\"78hj80j9-d61e-4d4f-be78-181834eeff6d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"sb-telemetry-master.1\",\n \"version_key\": \"1713442037275\"\n }\n}" - }, - { - "name": "Failure: Dataset with given dataset_id already exists", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"master-dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": []\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Conflict", - "code": 409, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:56:06+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"76612ad1-241b-4ce3-8af4-88db860697f4\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset already exists\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset contains duplicate denorm out field", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T17:59:06+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"54c5b692-dc37-432e-b556-7f396d7c9e07\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DUPLICATE_DENORM_KEY_FOUND\",\n \"message\": \"Duplicate denorm key found\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid request payload provided", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\"eid\"]\n },\n \"additionalProperties\":true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T18:00:34+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"615c1e4c-8c19-44fd-b29c-c235e7cbb5f0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"SCHEMA_VALIDATION_FAILED\",\n \"message\": \"#additionalProperties should NOT have additional properties\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-w\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n \"tag1\",\n \"tag2\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/create" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-16T18:02:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"dd1c0e11-fb4c-484c-81fe-82c9e2eee053\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_CREATION_FAILURE\",\n \"message\": \"Failed to create dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Dataset update", - "protocolProfileBehavior": { - "disabledSystemHeaders": {} - }, - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": 1713465933736,\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update", - "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations" - }, - "response": [ - { - "name": "Updated successfully", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713465933736\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T00:16:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4b2c0c22-f765-46f9-80af-33d0442db5ca\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"sb-telemetry.1\",\n \"version_key\": \"1713465973004\"\n }\n}" - }, - { - "name": "Failure: Dataset does not exists to update", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"version_key\":\"78997788678\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:20:39+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ebcf0a62-7c95-40f5-adf2-87ab90a40a80\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists to update\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid payload provided", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"version_key\":\"1787838782\",\n \"name\": {},\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:22:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9408e137-ada8-48b8-99e9-90d5cdd35e35\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/name/type should be string\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: No field provided along with dataset_id", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713507455695\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:23:06+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"347f465f-f100-4095-9801-38c4943380c0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Cannot update as dataset not in draft state", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry2.1\",\n \"version_key\":\"17387881289\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:25:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a9dfc926-c893-4ec3-82b8-bc8601d928a9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE\",\n \"message\": \"Dataset cannot be updated as it is not in draft state\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset contains duplicate denorm out field", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713507462417\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"add\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key1\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:29:25+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"865b9bef-9b6d-467c-9065-6689126b0f42\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_DUPLICATE_DENORM_KEY\",\n \"message\": \"Dataset contains duplicate denorm out keys:[userdata]\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset tags to add already exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510056996\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:32:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e9ff0bc-d660-4c4e-a2b0-cb11246a6961\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TAGS_EXISTS\",\n \"message\": \"Dataset tags already exist\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset tags to remove do not exist", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510194427\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:33:33+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c29e7d0f-3eaf-4972-8804-c172db7b6f13\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TAGS_DO_NOT_EXIST\",\n \"message\": \"Dataset tags do not exist to remove\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset transformations to add already exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510277508\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:34:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e0a5cc26-032e-4be8-81ef-c7e76867c9c1\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TRANSFORMATIONS_EXIST\",\n \"message\": \"Dataset transformations already exists\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset transformations to update do not exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\":\"1713510277508\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key2\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"update\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:36:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0b5f515b-1958-4597-b603-b4038d4e5846\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TRANSFORMATIONS_DO_NOT_EXIST\",\n \"message\": \"Dataset transformations do not exist to update\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset transformations to remove do not exist", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510277508\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"key2\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n },\n \"action\": \"remove\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:37:20+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69d32411-4252-4b7f-8afa-6d436928c3d3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_TRANSFORMATIONS_DO_NOT_EXIST\",\n \"message\": \"Dataset transformations do not exist to remove\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset denorm fields to add already exists", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510527662\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:38:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0b807580-a2b5-4a44-adc4-80b35b4b6529\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_DENORM_EXISTS\",\n \"message\": \"Denorm fields already exist\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset denorm to remove does not exist", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"1713510527662\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"cdata\"\n },\n \"action\": \"remove\"\n }\n ]\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:40:07+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ce78e393-2b74-46da-a70b-2abd06ad3270\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_DENORM_DO_NOT_EXIST\",\n \"message\": \"Denorm fields do not exist to remove\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry.1\",\n \"version_key\": \"17987837982\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n \"action\": \"remove\"\n },\n {\n \"values\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n },\n \"action\": \"add\"\n }\n ]\n },\n \"transformation_config\": [\n {\n \"values\": {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n \"action\": \"add\"\n }\n ],\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\"\n },\n \"tags\": [\n {\n \"values\": [\n \"tag1\",\n \"tag2\"\n ],\n \"action\": \"remove\"\n },\n {\n \"values\": [\n \"tag3\",\n \"tag4\"\n ],\n \"action\": \"add\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/update" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:41:33+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"451daa85-9cb8-459a-b681-a256c6e912f0\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_FAILURE\",\n \"message\": \"Failed to update dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Dataset read", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - } - ] - }, - "description": "This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status \"Live\". \n \nValid fields that user can request are \"dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date\"" - }, - "response": [ - { - "name": "Read Draft dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:43:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"d02c643a-d51d-4c67-8a42-9f43e61f459e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"sb-telemetry.1\",\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"\"\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [\n \"tag3\",\n \"tag4\"\n ],\n \"status\": \"Draft\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": null,\n \"client_state\": {},\n \"version_key\": \"1713510527662\",\n \"created_date\": \"2024-04-18T23:45:00.389Z\",\n \"updated_date\": \"2024-04-19T01:38:47.670Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"eid\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n }\n ]\n }\n}" - }, - { - "name": "Read Live dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v1/datasets/read/telemetry01" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:44:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3d0efbc5-dea8-4815-a749-b9f9ae16ccfe\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry01\",\n \"dataset_id\": \"telemetry01\",\n \"type\": \"dataset\",\n \"name\": \"telemetry01\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T06:27:37.557Z\",\n \"updated_date\": \"2024-04-11T06:27:37.557Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"mid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ],\n \"version\": 1\n }\n}" - }, - { - "name": "Read specific fields from the dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft&fields=name,type,id", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "name,type,id" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:45:44+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da752dfe-0d88-4dd6-a6a8-d858f5960f7e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"id\": \"sb-telemetry.1\"\n }\n}" - }, - { - "name": "Read version_key from the dataset", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft&fields=version_key", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "version_key" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:47:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"a3ad1e17-671f-4294-a834-2ee3f255c9b3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1713510527662\"\n }\n}" - }, - { - "name": "Failure: Invalid field name provided", - "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v1/datasets/read/sb-telemetry.1?status=Draft&fields=newname", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v1", - "datasets", - "read", - "sb-telemetry.1" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "newname" - } - ] - } - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:46:03+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"302fec39-5070-4464-b02f-56b1d0418147\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset not found", - "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v1/datasets/read/new" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:46:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id not found\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v1/datasets/read/sb-telemetry.1" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:46:34+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"29d82020-101d-4506-8fb8-6f3603057064\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_READ_FAILURE\",\n \"message\": \"Failed to read dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Dataset List", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\":\"master-dataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list", - "description": "This API allows you to list all datasets. User can apply filters on dataset status and type, sort the datasets as per requested order." - }, - "response": [ - { - "name": "Dataset list success when no filters provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:54:56+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e7a2c5c1-5aa9-4422-81a5-6f197d3b689e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"telemetry01\",\n \"dataset_id\": \"telemetry01\",\n \"type\": \"dataset\",\n \"name\": \"telemetry01\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T06:27:37.557Z\",\n \"updated_date\": \"2024-04-11T06:27:37.557Z\",\n \"version\": 1,\n \"transformations_config\": [\n {\n \"field_key\": \"mid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ]\n },\n {\n \"id\": \"master-telemetrry\",\n \"dataset_id\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"name\": \"master-telemetrry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T09:16:36.904Z\",\n \"updated_date\": \"2024-04-11T09:16:36.904Z\",\n \"version\": 1\n },\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n },\n {\n \"id\": \"two.1\",\n \"dataset_id\": \"two\",\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"\"\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [\n \"tag3\",\n \"tag4\"\n ],\n \"status\": \"Draft\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": null,\n \"client_state\": {},\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-15T02:22:48.270Z\",\n \"updated_date\": \"2024-04-15T02:24:03.308Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"eid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n },\n {\n \"field_key\": \"cid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"eid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ]\n },\n {\n \"id\": \"telemetry01.1\",\n \"dataset_id\": \"telemetry01\",\n \"name\": \"telemetry01\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T00:57:37.581Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"old\",\n \"name\": \"j\",\n \"skill_id\": \"abc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 3\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T00:54:06.404Z\",\n \"updated_date\": \"2024-04-11T06:28:24.526Z\"\n },\n {\n \"id\": \"sb-telemetry.1\",\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"required\": [\n \"eid\"\n ]\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"mid\"\n },\n {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"edata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"\"\n },\n \"dataset_config\": {\n \"data_key\": \"mid\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [\n \"tag3\",\n \"tag4\"\n ],\n \"status\": \"Draft\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": null,\n \"client_state\": {},\n \"version_key\": \"1713510527662\",\n \"created_date\": \"2024-04-18T23:45:00.389Z\",\n \"updated_date\": \"2024-04-19T01:38:47.670Z\",\n \"transformations_config\": [\n {\n \"field_key\": \"eid\",\n \"transformation_function\": {},\n \"mode\": \"Strict\",\n \"metadata\": {}\n }\n ]\n }\n ],\n \"count\": 6\n }\n}" - }, - { - "name": "Dataset list success when status filter provided as array", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:55:39+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"836f853f-9ad8-4567-a9bb-74c5d3bba25d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"telemetry01\",\n \"dataset_id\": \"telemetry01\",\n \"type\": \"dataset\",\n \"name\": \"telemetry01\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T06:27:37.557Z\",\n \"updated_date\": \"2024-04-11T06:27:37.557Z\",\n \"version\": 1,\n \"transformations_config\": [\n {\n \"field_key\": \"mid\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"condition\": null\n },\n \"mode\": \"Strict\",\n \"metadata\": {\n \"_transformationType\": \"mask\",\n \"_transformedFieldDataType\": \"string\",\n \"_transformedFieldSchemaType\": \"string\",\n \"section\": \"transformation\"\n }\n }\n ]\n },\n {\n \"id\": \"master-telemetrry\",\n \"dataset_id\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"name\": \"master-telemetrry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T09:16:36.904Z\",\n \"updated_date\": \"2024-04-11T09:16:36.904Z\",\n \"version\": 1\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Dataset list success when status filter provided as string", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Publish\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:56:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"469ef16b-4211-4b71-b063-bbd81cc9df02\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n },\n {\n \"id\": \"telemetry01.1\",\n \"dataset_id\": \"telemetry01\",\n \"name\": \"telemetry01\",\n \"type\": \"dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"telemetry01\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T00:57:37.581Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"old\",\n \"name\": \"j\",\n \"skill_id\": \"abc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 3\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T00:54:06.404Z\",\n \"updated_date\": \"2024-04-11T06:28:24.526Z\"\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Dataset list success when type filter provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Publish\",\n \"type\": \"master-dataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:57:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d9aeb08-2952-485f-9944-4d118da7842e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n }\n ],\n \"count\": 1\n }\n}" - }, - { - "name": "Dataset list success based on sortBy values", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"master-dataset\"\n },\n \"sortBy\": [\n {\n \"field\": \"updated_date\",\n \"order\": \"asc\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"270a4e50-f7e5-4562-ae75-7a967866d065\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"master-telemetrry\",\n \"dataset_id\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"name\": \"master-telemetrry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"Skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-11T09:16:36.904Z\",\n \"updated_date\": \"2024-04-11T09:16:36.904Z\",\n \"version\": 1\n },\n {\n \"id\": \"master-telemetrry.1\",\n \"dataset_id\": \"master-telemetrry\",\n \"name\": \"master-telemetrry\",\n \"type\": \"master-dataset\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"extraction_key\": \"\",\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"skill\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"skill_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"master-telemetrry\"\n },\n \"dataset_config\": {\n \"data_key\": \"skill\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 4\n },\n \"tags\": [],\n \"status\": \"Publish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2024-04-11T03:46:36.938Z\",\n \"client_state\": {\n \"metadata\": {\n \"conflicts\": {},\n \"event\": {\n \"mergedEvent\": {\n \"skill\": \"new\",\n \"name\": \"j\",\n \"skill_id\": \"aabc\",\n \"Skill_id\": \"bbc\"\n }\n },\n \"activePage\": 1\n },\n \"pages\": {}\n },\n \"version_key\": \"1713509698978\",\n \"created_date\": \"2024-04-11T03:45:57.251Z\",\n \"updated_date\": \"2024-04-11T09:21:11.167Z\"\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Failure: Invalid payload provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"f9e1fae9-4660-409d-bb4d-e29888983d4b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/additionalProperties should NOT have additional properties\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to database failed", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v1/datasets/list" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:59:15+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"81c3a4a5-e8ac-41de-be00-6d0300ecb106\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_FAILURE\",\n \"message\": \"Failed to list dataset\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ], - "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." - }, - { - "name": "Data Ingest", - "item": [ - { - "name": "Data ingest", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": {\n \"event\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/sb-telemetry-imported" - }, - "response": [ - { - "name": "Failure: schema validation", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/test2" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "300" - }, - { - "key": "ETag", - "value": "W/\"12c-KlibM35IozkQCEPK3Z48aogSrW0\"" - }, - { - "key": "Date", - "value": "Wed, 17 Apr 2024 09:20:53 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-17T14:50:53+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"98020cf9-4239-4b6e-beea-c687d21fafa3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Data ingest success (individual)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/test2" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "261" - }, - { - "key": "ETag", - "value": "W/\"105-27xQZBGpuJMZPC95g89XsOzED1w\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:15:09 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:45:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2de69987-4f6b-4c31-b3f1-a68018ddfabf\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" - }, - { - "name": "Data ingest successful (batch)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/test2" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "261" - }, - { - "key": "ETag", - "value": "W/\"105-mFJEsmzd/2rti3grZtWjHTnizDo\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:15:54 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:45:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"48ff1bcf-5bd3-487d-adf8-e1066d2f9638\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" - }, - { - "name": "Dataset not found", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/added-tags" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "317" - }, - { - "key": "ETag", - "value": "W/\"13d-lTx0GQxTBAUVJyB0HvmIfDIebZg\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:16:31 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:46:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"bc678713-fcc3-40fc-8ead-22b1a80c381f\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Entry topic not found", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/in/sample-test" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "316" - }, - { - "key": "ETag", - "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 10:17:00 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data query", - "item": [ - { - "name": "Data query", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"granularity\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"rollups-configured\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query" - }, - "response": [ - { - "name": "Success: SQL query", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"test\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/test" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "538" - }, - { - "key": "ETag", - "value": "W/\"21a-K+pNBJkYiMniehzSbVK9ufWWRLY\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:04:27 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:34:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"6b853b3a-fe62-4140-9b05-f2c6a6ba45bc\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-16T13:59:22.322Z\",\n \"school_id\": 10021,\n \"school_category\": \"secondary\",\n \"grade\": 1,\n \"gender\": \"others\",\n \"total_students\": 694,\n \"students_marked\": 694,\n \"students_present\": 137,\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null\n }\n ]\n}" - }, - { - "name": "SQL Failure: Invalid date range", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"telemetry-events\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-events" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "362" - }, - { - "key": "ETag", - "value": "W/\"16a-FIy8wK3EfIWVpWBvP7qSM0cm6v4\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:06:32 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:36:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"5aaf81d1-675d-4071-883a-f92d96656aa4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "SQL Failure: dataset not found in druid", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"taxt_trip\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"tests\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/taxt_trip" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "347" - }, - { - "key": "ETag", - "value": "W/\"15b-vsW6KGK4v2u3JeTsm3XZSN6/dqk\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:07:20 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:37:20+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"14594ee4-a009-44a4-8b61-1cdf4b8f60ad\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource taxt_trip.1_day not available for querying\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "SQL Failure: Datasource not found in live table", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"undefined\",\n \"table\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/undefined" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "341" - }, - { - "key": "ETag", - "value": "W/\"155-YXF5IPOdMM4KDIobS8ujcXwYyJs\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:08:53 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:38:53+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19c4e6be-e6ec-4d0d-a027-4e5ebc23889f\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Success: native query", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\": \"test\",\n \"table\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2020-12-31/2024-01-21\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/test" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "289" - }, - { - "key": "ETag", - "value": "W/\"121-KL/17PzocISF9bLwdErxBDCblqU\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:09:42 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:39:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2e2b3931-0f07-4536-a9f1-7123e500dabb\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-16T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 2\n }\n }\n ]\n}" - }, - { - "name": "Failure: Datasource not found in live table (native)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-eventssss" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "351" - }, - { - "key": "ETag", - "value": "W/\"15f-gaXKko6YU3dRhaApoksdaWjOP0k\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:11:25 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:41:25+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"1ffd02b2-6123-4682-880e-354a7005c060\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid date range (native)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"telemetry-events\",\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-events" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "362" - }, - { - "key": "ETag", - "value": "W/\"16a-sXmpPCkBQ3h2VWGFQFLg9z5TOAA\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:12:19 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:42:19+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f4bd3648-d8fd-4187-a860-3173626756d5\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Dataset not found in druid", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "name": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"1.0\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"datasetId\":\"telemetry-events\",\n \"granularity\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:4444/v1/data/query/telemetry-events" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "348" - }, - { - "key": "ETag", - "value": "W/\"15c-9MPd/+st2IWS5BfmezwbSXOEK2Q\"" - }, - { - "key": "Date", - "value": "Thu, 18 Apr 2024 12:13:36 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-18T17:43:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4c5a9e26-9395-406a-95f3-2d17afe4445d\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-events not available for querying\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data exhaust", - "item": [ - { - "name": "Data exhaust", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:4444/v1/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", - "host": [ - "localhost" - ], - "port": "4444", - "path": [ - "v1", - "data", - "exhaust", - "beckn-test-data" - ], - "query": [ - { - "key": "from", - "value": "2024-02-01" - }, - { - "key": "to", - "value": "2024-02-22" - }, - { - "key": "type", - "value": "raw" - } - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Query Templates", - "item": [ - { - "name": "New Request", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "text/plain" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"queryType\": \"scan\",\n \"dataSource\": {{DATASET}},\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": {{limits}}\n}", - "options": { - "raw": { - "language": "text" - } - } - }, - "url": { - "raw": "localhost:4444/v1/template/create/json_template_2?type=json", - "host": [ - "localhost" - ], - "port": "4444", - "path": [ - "v1", - "template", - "create", - "json_template_2" - ], - "query": [ - { - "key": "type", - "value": "json" - } - ] - } - }, - "response": [] - } - ] - } - ] -} \ No newline at end of file diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json similarity index 67% rename from api-service/postman-collection/updated_v2_collection.json rename to api-service/postman-collection/Obsrv v2 apis.postman_collection.json index 22412089..b48c505b 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -1,9 +1,11 @@ { "info": { - "_postman_id": "51471244-e9b6-453e-964d-dccbdac80e72", - "name": "V2 docs", + "_postman_id": "d3ffc748-3d71-4395-9298-e065845f7bfb", + "name": "Obsrv v2 apis", + "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "37164806" + "_exporter_id": "26192103", + "_collection_link": "https://speeding-star-177775.postman.co/workspace/2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" }, "item": [ { @@ -21,14 +23,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create", + "description": "This API allows you to create new datasets used by the analytical data source." }, "response": [ { @@ -43,14 +46,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -86,7 +89,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" }, { "name": "Success: Master dataset created successfully", @@ -100,14 +103,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -143,7 +146,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" }, { "name": "Failure: Master dataset already exists", @@ -157,14 +160,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "Conflict", "code": 409, @@ -200,7 +203,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" }, { "name": "Failure: Dataset already exists", @@ -214,14 +217,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "Conflict", "code": 409, @@ -257,7 +260,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" }, { "name": "Failure: Invalid request body", @@ -271,14 +274,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "Bad Request", "code": 400, @@ -314,7 +317,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" }, { "name": "Success: Minimal request body", @@ -328,14 +331,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -371,7 +374,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" }, { "name": "Success: Dataset created successfully with all fields", @@ -385,14 +388,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/create" + "url": "localhost:3000/v2/datasets/create" }, "status": "OK", "code": 200, @@ -428,7 +431,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" } ] }, @@ -444,14 +447,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url", + "description": "This API generates presigned URLs to upload or download files from cloud" }, "response": [ { @@ -466,14 +470,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "OK", "code": 200, @@ -509,7 +513,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" }, { "name": "Success: Generate get url", @@ -524,14 +528,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "OK", "code": 200, @@ -567,7 +571,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" }, { "name": "Failure: limit exceeds", @@ -582,14 +586,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "Bad Request", "code": 400, @@ -625,7 +629,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" }, { "name": "Failure: Invalid request", @@ -640,14 +644,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/files/generate-url" + "url": "localhost:3000/v2/files/generate-url" }, "status": "Bad Request", "code": 400, @@ -683,250 +687,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Data schema generator", - "request": { - "method": "POST", - "header": [ - { - "key": "Accept", - "value": "application/json, text/plain, */*", - "disabled": true - }, - { - "key": "Accept-Language", - "value": "en-GB,en", - "disabled": true - }, - { - "key": "Cache-Control", - "value": "no-store", - "disabled": true - }, - { - "key": "Connection", - "value": "keep-alive", - "disabled": true - }, - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", - "disabled": true - }, - { - "key": "Origin", - "value": "http://localhost:3001", - "disabled": true - }, - { - "key": "Pragma", - "value": "no-store", - "disabled": true - }, - { - "key": "Referer", - "value": "http://localhost:3001/console/dataset/new", - "disabled": true - }, - { - "key": "Sec-Fetch-Dest", - "value": "empty", - "disabled": true - }, - { - "key": "Sec-Fetch-Mode", - "value": "cors", - "disabled": true - }, - { - "key": "Sec-Fetch-Site", - "value": "same-origin", - "disabled": true - }, - { - "key": "Sec-GPC", - "value": "1", - "disabled": true - }, - { - "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "disabled": true - }, - { - "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", - "disabled": true - }, - { - "key": "sec-ch-ua-mobile", - "value": "?0", - "disabled": true - }, - { - "key": "sec-ch-ua-platform", - "value": "\"macOS\"", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3007/dataset/v1/dataschema" - }, - "response": [ - { - "name": "Success: Schema generated successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Accept", - "value": "application/json, text/plain, */*", - "disabled": true - }, - { - "key": "Accept-Language", - "value": "en-GB,en", - "disabled": true - }, - { - "key": "Cache-Control", - "value": "no-store", - "disabled": true - }, - { - "key": "Connection", - "value": "keep-alive", - "disabled": true - }, - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", - "disabled": true - }, - { - "key": "Origin", - "value": "http://localhost:3001", - "disabled": true - }, - { - "key": "Pragma", - "value": "no-store", - "disabled": true - }, - { - "key": "Referer", - "value": "http://localhost:3001/console/dataset/new", - "disabled": true - }, - { - "key": "Sec-Fetch-Dest", - "value": "empty", - "disabled": true - }, - { - "key": "Sec-Fetch-Mode", - "value": "cors", - "disabled": true - }, - { - "key": "Sec-Fetch-Site", - "value": "same-origin", - "disabled": true - }, - { - "key": "Sec-GPC", - "value": "1", - "disabled": true - }, - { - "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "disabled": true - }, - { - "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", - "disabled": true - }, - { - "key": "sec-ch-ua-mobile", - "value": "?0", - "disabled": true - }, - { - "key": "sec-ch-ua-platform", - "value": "\"macOS\"", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3007/dataset/v1/dataschema" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4414" - }, - { - "key": "ETag", - "value": "W/\"113e-ykaeY2EqBHdGqGLcr7K3WSs5fYo\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:28:22 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" } ] }, @@ -942,14 +703,15 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update", + "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors" }, "response": [ { @@ -964,14 +726,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "OK", "code": 200, @@ -1007,7 +769,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" }, { "name": "Success: Updated successfully", @@ -1021,14 +783,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "OK", "code": 200, @@ -1064,7 +826,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" }, { "name": "Failure: Outdated key provided", @@ -1078,14 +840,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Conflict", "code": 409, @@ -1121,7 +883,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" }, { "name": "Failure: Dataset not exists to update", @@ -1135,14 +897,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Not Found", "code": 404, @@ -1178,7 +940,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" }, { "name": "Failure: Invalid request", @@ -1192,14 +954,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Bad Request", "code": 400, @@ -1235,7 +997,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" }, { "name": "Failure: No fields are provided to update", @@ -1249,14 +1011,14 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/update" + "url": "localhost:3000/v2/datasets/update" }, "status": "Bad Request", "code": 400, @@ -1292,7 +1054,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" } ] }, @@ -1307,11 +1069,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "raw": "localhost:3000/v2/datasets/read/beckn-test-data?mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1324,7 +1086,8 @@ "value": "edit" } ] - } + }, + "description": "This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status \"Live\". This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, it creates one using the live dataset and returns the dataset details." }, "response": [ { @@ -1337,7 +1100,7 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3007/v2/datasets/read/master-test" + "url": "localhost:3000/v2/datasets/read/master-test" }, "status": "OK", "code": 200, @@ -1373,7 +1136,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" }, { "name": "Success: Read draft dataset", @@ -1386,11 +1149,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "raw": "localhost:3000/v2/datasets/read/beckn-test-data?mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1439,7 +1202,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" }, { "name": "Success: Read specific column", @@ -1452,11 +1215,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "raw": "localhost:3000/v2/datasets/read/master-test?status=Draft&fields=name,type,id", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1509,7 +1272,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" }, { "name": "Success: Read version_key", @@ -1522,11 +1285,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", + "raw": "localhost:3000/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1579,7 +1342,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" }, { "name": "Success: Read from draft, if not present cerate draft from live dataset and then read", @@ -1592,11 +1355,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/sample1?mode=edit", + "raw": "localhost:3000/v2/datasets/read/sample1?mode=edit", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1645,7 +1408,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" }, { "name": "Failure: Invalid field name provided", @@ -1658,11 +1421,11 @@ } ], "url": { - "raw": "localhost:3007/v2/datasets/read/telemetry_record?fields=newname", + "raw": "localhost:3000/v2/datasets/read/telemetry_record?fields=newname", "host": [ "localhost" ], - "port": "3007", + "port": "3000", "path": [ "v2", "datasets", @@ -1711,7 +1474,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" }, { "name": "Failure: Dataset not found", @@ -1723,7 +1486,7 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3007/v2/datasets/read/new_telemetry_record.1" + "url": "localhost:3000/v2/datasets/read/new_telemetry_record.1" }, "status": "Not Found", "code": 404, @@ -1759,7 +1522,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" } ] }, @@ -1770,14 +1533,15 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list", + "description": "This API allows you to list all datasets. User can apply filters on dataset status and type." }, "response": [ { @@ -1787,14 +1551,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1830,7 +1594,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" }, { "name": "Success: Filter based on status as array", @@ -1839,14 +1603,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1882,7 +1646,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" }, { "name": "Success: Filter basen on status as string", @@ -1891,14 +1655,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1934,7 +1698,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" }, { "name": "Success: Filter based on dataset type", @@ -1943,14 +1707,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "OK", "code": 200, @@ -1986,7 +1750,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" }, { "name": "Failure: Invalid payload", @@ -1995,14 +1759,14 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3007/v2/datasets/list" + "url": "localhost:3000/v2/datasets/list" }, "status": "Bad Request", "code": 400, @@ -2038,7 +1802,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" } ] }, @@ -2065,7 +1829,8 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema", + "description": "This api is used to generate data schema for the given dataset event." }, "response": [ { @@ -2091,7 +1856,7 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema" }, "status": "OK", "code": 200, @@ -2127,7 +1892,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" }, { "name": "Failure: Invalid request body", @@ -2152,7 +1917,7 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema" }, "status": "Bad Request", "code": 400, @@ -2188,7 +1953,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" }, { "name": "Failure: Invalid request (config not provided)", @@ -2213,7 +1978,7 @@ } } }, - "url": "localhost:3007/v2/datasets/dataschema" + "url": "localhost:3000/v2/datasets/dataschema" }, "status": "Bad Request", "code": 400, @@ -2249,175 +2014,2857 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" } ] - } - ] - }, - { - "name": "Connector api's", - "item": [ + }, { - "name": "Connector list", + "name": "Dataset Status Transition", "request": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-events\",\n \"status\": \"ReadyToPublish\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition", + "description": "This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset." }, "response": [ { - "name": "Failure: Invalid request body, filter option array should not be empty", + "name": "Delete success: Deleted dataset successfully", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"trip-data\",\n \"status\": \"Delete\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "399" - }, - { - "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], + "header": [], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset deleted successfully\",\n \"dataset_id\": \"master.1\"\n }\n}" }, { - "name": "Success: Filtered based on status", + "name": "Delete failure: Dataset not found to delete", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"master\",\n \"status\": \"Delete\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "250" - }, - { - "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], + "header": [], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to delete\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Filtered based on category", + "name": "Live success: Dataset published successfully", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record\",\n \"status\": \"Live\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/datasets/status-transition" }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset published successfully\",\n \"dataset_id\": \"telemetry\"\n }\n}" + }, + { + "name": "Live failure: Dataset not found to publish", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-data\",\n \"status\": \"Live\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to publish\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Live failure: Dataset in draft state", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry2\",\n \"status\": \"Live\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_NOT_READY_FOR_PUBLISH\",\n \"message\": \"Failed to publish dataset as it is in draft state\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "ReadyToPublish success: Dataset is ready to publish", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-events\",\n \"status\": \"ReadyToPublish\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:30:04+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"84858e85-6a97-43cb-b8e9-17a7e0a43365\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to ReadyToPublish successful\",\n \"dataset_id\": \"telemetry-events\"\n }\n}" + }, + { + "name": "ReadyToPublish failure: Incomplete dataset configs", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record\",\n \"status\": \"ReadyToPublish\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:36:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"a504565b-41ff-4c0f-9d64-f96df9ed89bb\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_CONFIGS_INVALID\",\n \"message\": \"#properties/denorm_config/properties/denorm_fields/items/required must have required property 'dataset_name'\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "ReadyToPublish failure: Dataset not in draft state", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry-events\",\n \"status\": \"ReadyToPublish\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Failed to mark dataset Ready to publish as it not in draft state\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Retire success: Dataset retired successfully", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset retired successfully\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" + }, + { + "name": "Retire Failure: Dataset not found to retire", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"sb-telemetry2\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to retire\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Retire Failure: Dataset is already retired", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"master-telemetrry\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_ALREADY_RETIRED\",\n \"message\": \"Dataset is already retired\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Retire failure: Cannot retire master dataset as it is used by other datasets", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Retire\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is used by other datasets\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid request payload provided", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Live\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:03:56+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"ba4c86bd-b438-4582-b178-2410a5c5dd15\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_STATUS_INVALID_INPUT\",\n \"message\": \"#properties/request/properties/status/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Connection to the db failed", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Delete\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/status-transition" + }, + "status": "Internal Server Error", + "code": 500, + "_postman_previewlanguage": "json", + "header": [], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:24:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"92928434-719f-47d4-9946-1e40ecd53253\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"DATASET_STATUS_FAILURE\",\n \"message\": \"Failed to perform status transition on datasets\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Dataset import", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?override=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "override", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Dataset export", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [] + }, + { + "name": "Dataset copy", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/datasets/copy" + }, + "response": [] + } + ], + "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "response": [ + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "399" + }, + { + "key": "ETag", + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:43:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + }, + { + "name": "Success: Filtered based on status", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "250" + }, + { + "key": "ETag", + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:55:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + }, + { + "name": "Success: Filtered based on category", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2571" + }, + { + "key": "ETag", + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:25:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + }, + { + "name": "Success: Connectors list with all filter options", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4380" + }, + { + "key": "ETag", + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:26:32 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Connectors list without filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4380" + }, + { + "key": "ETag", + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:27:37 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "response": [ + { + "name": "Success: Read live Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "3304" + }, + { + "key": "ETag", + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 12:47:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + }, + { + "name": "Success: Read Draft Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "mssql-connector-2.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "744" + }, + { + "key": "ETag", + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 07:17:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + }, + { + "name": "Failure: Connector not found", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "276" + }, + { + "key": "ETag", + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 09:32:48 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Ingest", + "item": [ + { + "name": "Data ingest", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sb-telemetry-imported" + }, + "response": [ + { + "name": "Entry topic not found", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sample-test" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "316" + }, + { + "key": "ETag", + "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" + }, + { + "key": "Date", + "value": "Thu, 18 Apr 2024 10:17:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: schema validation", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/test2" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "300" + }, + { + "key": "ETag", + "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:03:32 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: Data ingested successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "261" + }, + { + "key": "ETag", + "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:05:48 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + }, + { + "name": "Success: Data ingested successfully (batch)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/test2" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "261" + }, + { + "key": "ETag", + "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:06:49 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + }, + { + "name": "Failure: Dataset not found", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/added-tags" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "317" + }, + { + "key": "ETag", + "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:07:29 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Entry topic not found", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sample1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "316" + }, + { + "key": "ETag", + "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:09:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data query", + "item": [ + { + "name": "Data query", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "response": [ + { + "name": "Success: native query (interval as array type)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "313" + }, + { + "key": "ETag", + "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:17:45 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + }, + { + "name": "Success: native query (queryType: scan)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "313" + }, + { + "key": "ETag", + "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:21:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + }, + { + "name": "Success: native query (normal intervals & queryType : timeseries)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "289" + }, + { + "key": "ETag", + "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:21:49 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + }, + { + "name": "Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/telemetry-eventssss" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "356" + }, + { + "key": "ETag", + "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:13:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid date range", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/telemetry-events" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:16:29 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "356" + }, + { + "key": "ETag", + "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:20:27 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: sql query", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "582" + }, + { + "key": "ETag", + "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:12:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + }, + { + "name": "SQL Failure: dataset not found in druid", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "356" + }, + { + "key": "ETag", + "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:22:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "SQL Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/undefined" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "346" + }, + { + "key": "ETag", + "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:23:02 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data exhaust", + "item": [ + { + "name": "Data exhaust", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "exhaust", + "beckn-test-data" + ], + "query": [ + { + "key": "from", + "value": "2024-02-01" + }, + { + "key": "to", + "value": "2024-02-22" + }, + { + "key": "type", + "value": "raw" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Query Templates", + "item": [ + { + "name": "Read template", + "request": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql_template_11" + }, + "response": [ + { + "name": "Success: JSON template", + "originalRequest": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/jsontemplate111" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "537" + }, + { + "key": "ETag", + "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:24:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" + }, + { + "name": "Success: SQL template", + "originalRequest": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql1" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "528" + }, + { + "key": "ETag", + "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:24:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" + }, + { + "name": "Failure : Template not exists", + "originalRequest": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql100" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "294" + }, + { + "key": "ETag", + "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:25:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Delete template", + "request": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/yashash-k" + }, + "response": [ + { + "name": "Success: Deleted successfully", + "originalRequest": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/yashash-k" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "241" + }, + { + "key": "ETag", + "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:28:27 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" + }, + { + "name": "Failure: Template does not exists", + "originalRequest": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/json_template" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "303" + }, + { + "key": "ETag", + "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:28:41 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Create query template", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create/json_template_111" + }, + "response": [ + { + "name": "Success : Create query template success (json)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "359" + }, + { + "key": "ETag", + "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 16:59:01 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + }, + { + "name": "Success: Create query template success (SQL)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "339" + }, + { + "key": "ETag", + "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 16:59:58 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + }, + { + "name": "Failure: Template already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "350" + }, + { + "key": "ETag", + "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:03:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Missing required variables", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "407" + }, + { + "key": "ETag", + "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:05:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Schema validation failure", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "416" + }, + { + "key": "ETag", + "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:13:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "List templates", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "response": [ + { + "name": "empty request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "6864" + }, + { + "key": "ETag", + "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 13:46:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + }, + { + "name": "Success : order by", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "7700" + }, + { + "key": "ETag", + "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:06:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : Filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2708" + }, + { + "key": "ETag", + "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : limit and offset", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1921" + }, + { + "key": "ETag", + "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:11:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" + } + ] + }, + { + "name": "update template", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "response": [ + { + "name": "Success: update successful", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "314" + }, + { + "key": "ETag", + "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:51:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + }, + { + "name": "Failure: required variables not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "429" + }, + { + "key": "ETag", + "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:53:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Template name validation failure", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "405" + }, + { + "key": "ETag", + "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:56:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query_type should when updating query", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, { "key": "Content-Type", "value": "application/json; charset=utf-8" }, { "key": "Content-Length", - "value": "2571" + "value": "412" }, { "key": "ETag", - "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:25:03 GMT" + "value": "Fri, 10 May 2024 07:02:57 GMT" }, { "key": "Connection", @@ -2429,26 +4876,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Connectors list with all filter options", + "name": "Failure: query should present when updating query_type", "originalRequest": { - "method": "POST", + "method": "PATCH", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/template/update/sql11template1" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -2461,15 +4908,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "412" }, { "key": "ETag", - "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:26:32 GMT" + "value": "Fri, 10 May 2024 07:04:55 GMT" }, { "key": "Connection", @@ -2481,26 +4928,122 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "query template", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "response": [ { - "name": "Success: Connectors list without filters", + "name": "Failure: invalid date range (native template)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "telemetry-events" + ], + "query": [ + { + "key": "limit", + "value": "1", + "disabled": true + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "336" + }, + { + "key": "ETag", + "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + }, + { + "key": "Date", + "value": "Thu, 02 May 2024 07:29:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": "localhost:3000/v2/template/query/sql_test_1" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -2513,15 +5056,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "357" }, { "key": "ETag", - "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" + "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:27:37 GMT" + "value": "Mon, 13 May 2024 07:51:46 GMT" }, { "key": "Connection", @@ -2533,27 +5076,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - } - ] - }, - { - "name": "Connector Read", - "request": { - "method": "GET", - "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" - }, - "response": [ + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, { - "name": "Success: Read live Connector", + "name": "Failure: Datasource not found in live table", "originalRequest": { - "method": "GET", + "method": "POST", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -2566,15 +5108,15 @@ }, { "key": "Content-Length", - "value": "3304" + "value": "343" }, { "key": "ETag", - "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 12:47:54 GMT" + "value": "Mon, 13 May 2024 07:53:47 GMT" }, { "key": "Connection", @@ -2586,33 +5128,81 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Read Draft Connector", + "name": "Failure: invalid date range", "originalRequest": { - "method": "GET", + "method": "POST", "header": [], - "url": { - "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "connectors", - "read", - "mssql-connector-2.0.0" - ], - "query": [ - { - "key": "mode", - "value": "edit" + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" } - ] + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:58:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: JSON template with request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/jsontemplate1" }, "status": "OK", "code": 200, @@ -2628,15 +5218,15 @@ }, { "key": "Content-Length", - "value": "744" + "value": "301" }, { "key": "ETag", - "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 07:17:12 GMT" + "value": "Tue, 14 May 2024 06:22:24 GMT" }, { "key": "Connection", @@ -2648,17 +5238,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" }, { - "name": "Failure: Connector not found", + "name": "Success: SQL template query with request body", "originalRequest": { - "method": "GET", + "method": "POST", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2671,15 +5270,15 @@ }, { "key": "Content-Length", - "value": "276" + "value": "594" }, { "key": "ETag", - "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 09:32:48 GMT" + "value": "Tue, 14 May 2024 06:03:06 GMT" }, { "key": "Connection", @@ -2691,7 +5290,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" } ] } diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index d0478474..2a0c64ed 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -6,8 +6,9 @@ import { getDatasourceList } from "../../services/DatasourceService"; import logger from "../../logger"; import { druidHttpService, getDatasourceListFromDruid } from "../../connections/druidConnection"; import { apiId } from "./DataOutController"; -import { ErrorObject } from "../../types/ResponseModel"; +import { Parser } from "node-sql-parser"; import { obsrvError } from "../../types/ObsrvError"; +const parser = new Parser(); const momentFormat = "YYYY-MM-DD HH:MM:SS"; let dataset_id: string; @@ -55,6 +56,27 @@ const getLimit = (queryLimit: number, maxRowLimit: number) => { return queryLimit > maxRowLimit ? maxRowLimit : queryLimit; }; +const parseSqlQuery = (queryPayload: any) => { + try { + const vocabulary: any = parser.astify(queryPayload?.query); + const isLimitIncludes = JSON.stringify(vocabulary); + if (_.includes(isLimitIncludes, "{{LIMIT}}")) { + return queryPayload?.query + } + const limit = _.get(vocabulary, "limit") + if (limit === null) { + _.set(vocabulary, "limit.value[0].value", queryRules.common.maxResultRowLimit) + _.set(vocabulary, "limit.value[0].type", "number") + let convertToSQL = parser.sqlify(vocabulary); + convertToSQL = convertToSQL.replace(/`/g, "\""); + queryPayload.query = convertToSQL + } + return true + } catch (error) { + logger.warn("Sql parser error", error) + return false + } +} const setQueryLimits = (queryPayload: any) => { if (_.isObject(queryPayload?.query)) { const threshold = _.get(queryPayload, "query.threshold") @@ -77,15 +99,18 @@ const setQueryLimits = (queryPayload: any) => { } if (_.isString(queryPayload?.query)) { - let query = queryPayload.query; - if (/\blimit\b/i.test(query)) { - return query; - } - const limitRegex = /\bLIMIT\b\s+\d+/i; - if (!limitRegex.test(query)) { - const maxLimit = _.get(queryRules, "common.maxResultRowLimit"); - query += ` LIMIT ${maxLimit}`; - queryPayload.query = query; + const validDruidSql = parseSqlQuery(queryPayload) + if (!validDruidSql) { + let query = queryPayload.query; + if (/\blimit\b/i.test(query)) { + return query; + } + const limitRegex = /\bLIMIT\b\s+\d+/i; + if (!limitRegex.test(query)) { + const maxLimit = _.get(queryRules, "common.maxResultRowLimit"); + query += ` LIMIT ${maxLimit}`; + queryPayload.query = query; + } } } } @@ -129,7 +154,7 @@ const validateDateRange = (fromDate: moment.Moment, toDate: moment.Moment, allow } else { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Data range cannnot be more than ${allowedRange} days.`, code: errCode.invalidDateRange }) - throw { message: `Invalid date range! make sure your range cannot be more than ${allowedRange} days`, statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange } as ErrorObject; + throw obsrvError("", errCode.invalidDateRange, `Invalid date range! make sure your range cannot be more than ${allowedRange} days`, "BAD_REQUEST", 400); } }; @@ -156,7 +181,7 @@ const getDataSourceRef = async (datasetId: string, requestGranularity?: string) const dataSources = await getDatasourceList(datasetId) if (_.isEmpty(dataSources)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Datasource ${datasetId} not available in datasource live table`, code: errCode.notFound }) - throw { message: `Datasource ${datasetId} not available for querying`, statusCode: 404, errCode: "NOT_FOUND", code: errCode.notFound } as ErrorObject; + throw obsrvError("", errCode.notFound, `Datasource ${datasetId} not available for querying`, "NOT_FOUND", 404); } const record = dataSources.find((record: any) => { const metadata = _.get(record, "dataValues.metadata", {}); @@ -191,7 +216,7 @@ const setDatasourceRef = async (datasetId: string, payload: any): Promise = if (!_.includes(existingDatasources.data, datasourceRef)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Dataset ${datasetId} with table ${granularity} is not available for querying`, code: errCode.notFound }) - throw { message: `Dataset ${datasetId} with table ${granularity} is not available for querying`, statusCode: 404, errCode: "NOT_FOUND", code: errCode.notFound } as ErrorObject; + throw obsrvError("", errCode.notFound, `Dataset ${datasetId} with table ${granularity} is not available for querying`, "NOT_FOUND", 404); } if (_.isString(payload?.query)) { payload.query = payload.query.replace(datasetId, datasourceRef) From 16b332ba713d081a56188574610aec354857338f Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 24 Oct 2024 18:01:32 +0530 Subject: [PATCH 186/311] #OBS-I285: fix: linting issue fix --- .../src/controllers/GenerateDataSchema/GenerateDataSchema.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts index 11bcee7e..ee96f725 100644 --- a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -46,7 +46,8 @@ const schemaGenerate = (sample: Map[], config: Record) result.schema = removeFormats(result.schema) return result } else { - let { schema, removedKeys } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + let { schema } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + const { removedKeys } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); schema = schemaArrayValidator.validate(schema) const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) rollupInfo = schemaCardinalityAnalyser.analyse() From 1287311371588f865d90ed6caf3e332024bee248 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 24 Oct 2024 18:16:08 +0530 Subject: [PATCH 187/311] #OBS-I285: fix: lint fix --- .../src/controllers/GenerateDataSchema/GenerateDataSchema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts index ee96f725..ce3c7bde 100644 --- a/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -46,8 +46,8 @@ const schemaGenerate = (sample: Map[], config: Record) result.schema = removeFormats(result.schema) return result } else { - let { schema } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); - const { removedKeys } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + // eslint-disable-next-line + let { schema, removedKeys } = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); schema = schemaArrayValidator.validate(schema) const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) rollupInfo = schemaCardinalityAnalyser.analyse() From c90606f1ee8767544d074b5498581dbb237aaa38 Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:38:30 +0530 Subject: [PATCH 188/311] #OBS-I285: fix: Logging errors in the middleware (#263) * #OBS-I285: fix: Logging errors in the middleware * #OBS-I285: fix: Logging errors by excluding sensitive info * #OBS-I285: fix: linting issue fix --- api-service/src/helpers/ResponseHandler.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3c1273e2..67332b37 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -5,6 +5,7 @@ import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/help import moment from "moment"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; +import logger from "../logger"; const ResponseHandler = { successResponse: (req: Request, res: Response, result: Result) => { @@ -26,6 +27,8 @@ const ResponseHandler = { errorResponse: (error: Record, req: Request, res: Response) => { const { statusCode, message, errCode, code = "INTERNAL_SERVER_ERROR", trace = "" } = error; + const sanitizedError = { ...error, proxyAuthKey: "REDACTED" }; + logger.error(sanitizedError) const { id, entity, body } = req as any; const msgid = _.get(body, ["params", "msgid"]) const resmsgid = _.get(res, "resmsgid") @@ -37,6 +40,8 @@ const ResponseHandler = { obsrvErrorResponse: (error: ObsrvError, req: Request, res: Response) => { const { statusCode, message, errCode, code = "INTERNAL_SERVER_ERROR", data } = error; + const sanitizedError = { ...error, proxyAuthKey: "REDACTED" }; + logger.error(sanitizedError) const { id, entity, body } = req as any; const msgid = _.get(body, ["params", "msgid"]) const resmsgid = _.get(res, "resmsgid") From 5a16257f6144b9d3e4e177fb9f84c81058ba7975 Mon Sep 17 00:00:00 2001 From: yashash <126703764+yashashkumar@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:59:03 +0530 Subject: [PATCH 189/311] #OBS-I289 : added route (#265) --- api-service/package.json | 1 + .../GenerateSignedURL/GenerateSignedURL.ts | 60 ++----------------- api-service/src/routes/Router.ts | 3 +- 3 files changed, 7 insertions(+), 57 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index 2b858f69..782cda64 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -61,6 +61,7 @@ "@babel/traverse": "7.23.2" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/chai": "^4.3.3", "@types/chai-as-promised": "^7.1.5", "@types/chai-spies": "^1.0.3", diff --git a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts index 6182f9b3..6454b1fa 100644 --- a/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts +++ b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts @@ -6,20 +6,20 @@ import logger from "../../logger"; import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import GenerateURL from "./GenerateSignedURLValidationSchema.json" -import { cloudProvider } from "../../services/CloudServices"; import { config } from "../../configs/Config"; import { URLAccess } from "../../types/SampleURLModel"; -import { v4 as uuidv4 } from "uuid"; -import path from "path"; +import { generatePreSignedUrl } from "./helper"; export const apiId = "api.files.generate-url" export const code = "FILES_GENERATE_URL_FAILURE" const maxFiles = config.presigned_url_configs.maxFiles +let containerType: string; const generateSignedURL = async (req: Request, res: Response) => { const requestBody = req.body const msgid = _.get(req, ["body", "params", "msgid"]); const resmsgid = _.get(res, "resmsgid"); + containerType = _.get(req, ["body", "request", "type"]); try { const isRequestValid: Record = schemaValidation(req.body, GenerateURL) if (!isRequestValid.isValid) { @@ -46,21 +46,7 @@ const generateSignedURL = async (req: Request, res: Response) => { errCode: "BAD_REQUEST" } as ErrorObject, req, res); } - - const { filesList, updatedFileNames } = transformFileNames(files, access) - logger.info(`Updated file names with path:${updatedFileNames}`) - - const urlExpiry: number = getURLExpiry(access) - const preSignedUrls = await Promise.all(cloudProvider.generateSignedURLs(config.cloud_config.container, updatedFileNames, access, urlExpiry)) - const signedUrlList = _.map(preSignedUrls, list => { - const fileNameWithUid = _.keys(list)[0] - return { - filePath: getFilePath(fileNameWithUid), - fileName: filesList.get(fileNameWithUid), - preSignedUrl: _.values(list)[0] - } - }) - + const signedUrlList = await generatePreSignedUrl(access, files, containerType) logger.info({ apiId, requestBody, msgid, resmsgid, response: signedUrlList, message: `Sample urls generated successfully for files:${files}` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: signedUrlList }) } catch (error: any) { @@ -74,44 +60,6 @@ const generateSignedURL = async (req: Request, res: Response) => { } } -const getFilePath = (file: string) => { - return `${config.cloud_config.container}/${config.presigned_url_configs.service}/user_uploads/${file}` -} - -const transformFileNames = (fileList: Array, access: string): Record => { - if (access === URLAccess.Read) { - return transformReadFiles(fileList) - } - return transformWriteFiles(fileList) -} - -const transformReadFiles = (fileNames: Array) => { - const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { - fileMap.set(file, file) - return getFilePath(file) - }) - return { filesList: fileMap, updatedFileNames } -} - -const transformWriteFiles = (fileNames: Array) => { - const fileMap = new Map(); - const updatedFileNames = _.map(fileNames, file => { - const uuid = uuidv4().replace(/-/g, "").slice(0, 6); - const ext = path.extname(file) - const baseName = path.basename(file, ext) - const updatedFileName = `${baseName}_${uuid}${ext}` - fileMap.set(updatedFileName, file) - return getFilePath(updatedFileName) - }) - return { filesList: fileMap, updatedFileNames } - -} - -const getURLExpiry = (access: string) => { - return access === URLAccess.Read ? config.presigned_url_configs.read_storage_url_expiry : config.presigned_url_configs.write_storage_url_expiry -} - const checkLimitExceed = (files: Array): boolean => { return _.size(files) > maxFiles } diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 40f1df02..1e39d942 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -31,6 +31,7 @@ import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; +import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; export const router = express.Router(); @@ -58,7 +59,7 @@ router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onReq router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsList); router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); - +router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) From 43a9992e359eee82188be483c9b996fab49244dd Mon Sep 17 00:00:00 2001 From: SurabhiAngadi <138881390+SurabhiAngadi@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:16:03 +0530 Subject: [PATCH 190/311] fix: Stop the connectors specific to the dataset. (#266) * fix: uninstall dataset specific spark jobs * fix: remove nested loops * fix: uninstall dataset specific spark jobs * fix: remove nested loops --- .../src/command/connector_command.py | 26 +++++++++---------- command-service/src/model/db_models.py | 1 + 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index aabb9db4..8f5b145e 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -38,12 +38,12 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) + self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"], active_connectors, dataset_id) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result - def _stop_connector_jobs(self, is_masterdata, namespace): + def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, dataset_id): print(f"Uninstalling jobs for {namespace}..") base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] @@ -61,13 +61,13 @@ def _stop_connector_jobs(self, is_masterdata, namespace): helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode() - for job in jobs.splitlines()[1:]: - release_name = job.split()[0] - if base_helm_chart in job: - print("Uninstalling job {0}".format(release_name)) + jobs = helm_ls_result.stdout.decode().splitlines()[1:] + job_names = {job.split()[0] for job in jobs if base_helm_chart in job} + spark_connector = {connector.id for connector in active_connectors if connector.connector_runtime == "spark"} + for release_name in spark_connector: + if release_name in job_names: + print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") helm_uninstall_cmd = [ "helm", "uninstall", @@ -81,12 +81,10 @@ def _stop_connector_jobs(self, is_masterdata, namespace): stderr=subprocess.PIPE, ) if helm_uninstall_result.returncode == 0: - print(f"Successfully uninstalled job {release_name}...") + print(f"Successfully uninstalled job '{release_name}'...") else: - print( - f"Error uninstalling job {release_name}: {helm_uninstall_result.stderr.decode()}" - ) - + print(f"Error uninstalling job '{release_name}': {helm_uninstall_result.stderr.decode()}") + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None for connector in active_connectors: @@ -277,7 +275,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): def _get_connector_details(self, dataset_id): active_connectors = [] query = f""" - SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version + SELECT ci.id, ci.connector_id, ci.dataset_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version FROM connector_instances ci JOIN connector_registry cr on ci.connector_id = cr.id WHERE ci.status= %s and ci.dataset_id = %s diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index eaeea86d..65abb534 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -121,6 +121,7 @@ class ConnectorRegsitryv2: class ConnectorInstance: id: str connector_id: str + dataset_id: str operations_config: dict connector_runtime: str connector_source: dict From f8d8309d8ec55e3b988f74261667e5c6d5b14283 Mon Sep 17 00:00:00 2001 From: Anand Parthasarathy Date: Tue, 29 Oct 2024 15:24:26 +0530 Subject: [PATCH 191/311] #OBS-I289 fix: Fix flink connector deployments --- .../helm-charts/flink-connector/values.yaml | 13 ++++--------- command-service/src/command/db_command.py | 3 +-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index 573952f1..ed51316d 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -242,15 +242,10 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom -flink_jobs: - kafka-connector-1-0-0: - registry: sanketikahub - repository: flink-connectors - tag: 1.1.0 - enabled: "false" - connector_id: "kafka-connector-1.0.0" - source: "kafka-connector-1.0.0" - main_program: "kafka-connector-1.0.0.jar" + +# override flink_jobs +# flink_jobs: + commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 09003482..01c7e333 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -260,7 +260,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): operations_config = connector_config.operations_config if connector_config.operations_config is not None else {} if connector_config.version == 'v2': params = ( - connector_config.id, + "{0}-{1}".format(dataset_id, connector_config.id), dataset_id, connector_config.connector_id, connector_config.connector_config, @@ -273,7 +273,6 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): current_timestamp, current_timestamp, current_timestamp, - connector_config.connector_config, json.dumps(connector_config.operations_config).replace("'", "''"), draft_dataset_record.get('updated_by'), From d4f0bf0c4ae07041b0731ce2fbcade6e89967996 Mon Sep 17 00:00:00 2001 From: Anand Parthasarathy Date: Wed, 30 Oct 2024 11:57:37 +0530 Subject: [PATCH 192/311] #OBS-I289 fix: Fix Spark Connector deployments --- .../spark-connector-cron/templates/_cron_release_name.tpl | 2 +- command-service/src/command/connector_command.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl index f4b82f3c..094968a5 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl +++ b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl @@ -1,4 +1,4 @@ {{- define "base.cronReleaseName" -}} - {{- $name := printf "%s--%s" .Chart.Name .Values.instance_id }} + {{- $name := printf "%s" .Values.instance_id }} {{- default .Values.instance_id $name }} {{- end }} \ No newline at end of file diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 8f5b145e..2acc6ed0 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -216,7 +216,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): result = None release_name = connector_instance.id connector_source = connector_instance.connector_source - schedule = connector_instance.operations_config["schedule"] + schedule = connector_instance.operations_config["operations_config_schedule"] schedule_configs = { "Hourly": "0 * * * *", # Runs at the start of every hour From 2900c631a8a2fb251ab0d2e17e1bc49c163bd81d Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 5 Nov 2024 14:57:43 +0530 Subject: [PATCH 193/311] #000: Open Telemetry Integration: Generation of system events as open telemetry format --- api-service/package.json | 10 +++ api-service/src/app.ts | 80 +++++++++++++++++++- api-service/src/configs/ConnectionsConfig.ts | 4 +- api-service/src/services/telemetry.ts | 2 +- 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index 2b858f69..761549f8 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -22,6 +22,16 @@ "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", "@jsonhero/schema-infer": "^0.1.5", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.53.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.53.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.53.0", + "@opentelemetry/resources": "^1.26.0", + "@opentelemetry/sdk-logs": "^0.53.0", + "@opentelemetry/sdk-metrics": "^1.26.0", + "@opentelemetry/sdk-node": "^0.53.0", + "@opentelemetry/sdk-trace-node": "^1.26.0", + "@opentelemetry/semantic-conventions": "^1.27.0", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 6615ad74..e61ea038 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -9,9 +9,87 @@ import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; import { interceptAuditEvents } from "./services/telemetry"; +import { OTelService } from "./otel/OTelService"; +import { LogRecord } from "@opentelemetry/sdk-logs"; + + const app: Application = express(); - +OTelService.init() +OTelService.log() + +const auditLog = { + "eid": "AUDIT", + "ets": 1729158293107, + "ver": "1.0.0", + "mid": "759b9471-b3bd-4818-89f5-cbdf5cdfc421", + "actor": { + "id": "SYSTEM", + "type": "User" + }, + "context": { + "env": "local", + "sid": "37229d4c-38a1-4aac-94cf-5fe34230fb1a", + "pdata": { + "id": "local.api.service", + "ver": "1.0" + } + }, + "object": {}, + "edata": { + "action": "dataset:create", + "props": [ + { + "property": "id", + "ov": null, + "nv": "api.data.in" + }, + { + "property": "ver", + "ov": null, + "nv": "v2" + }, + { + "property": "ts", + "ov": null, + "nv": "1711966306164" + }, + { + "property": "params", + "ov": null, + "nv": { + "msgid": "e180ecac-8f41-4f21-9a21-0b3a1a368917" + } + }, + { + "property": "data", + "ov": null, + "nv": { + "eid": "INTERACT", + "date": "2022-01-01", + "ver": "3.0", + "syncts": 1668591949682, + "ets": 1668591949682 + } + } + ], + "transition": { + "timeUnit": "ms", + "duration": 437, + "toState": "completed", + "fromState": "inprogress" + } + } +}; + +// Emit the audit log +OTelService.emitAuditLog(auditLog); +//const loggerInstance = OTelService.getLoggerProvider(); + + + + + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); diff --git a/api-service/src/configs/ConnectionsConfig.ts b/api-service/src/configs/ConnectionsConfig.ts index b21512c9..a12d0a56 100644 --- a/api-service/src/configs/ConnectionsConfig.ts +++ b/api-service/src/configs/ConnectionsConfig.ts @@ -5,8 +5,8 @@ export const connectionConfig = { postgres: { host: env.postgres_host || "localhost", port: env.postgres_port || 5432, - database: env.postgres_database || "obsrv", - username: env.postgres_username || "postgres", + database: env.postgres_database || "postgres", + username: env.postgres_username || "manjunathdavanam", password: env.postgres_password || "postgres", }, kafka: { diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index 8408dfb3..6f3a1670 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -50,7 +50,7 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { - send({ messages: [{ value: JSON.stringify(event) }] }, telemetryTopic).catch(console.log); + send(event, telemetryTopic).catch(console.log); } const transformProps = (body: Record) => { From 680fdcbd3e8243a97a22a1289c87a4f610f5e977 Mon Sep 17 00:00:00 2001 From: Santhosh Date: Tue, 5 Nov 2024 17:37:08 +0530 Subject: [PATCH 194/311] #OBS-I307 - revert the code as this is fixed from the front end (#268) --- command-service/src/command/connector_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 2acc6ed0..8f5b145e 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -216,7 +216,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): result = None release_name = connector_instance.id connector_source = connector_instance.connector_source - schedule = connector_instance.operations_config["operations_config_schedule"] + schedule = connector_instance.operations_config["schedule"] schedule_configs = { "Hourly": "0 * * * *", # Runs at the start of every hour From 0a9236b1c9da0f611b675a4c23260d47eeda5dc1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 6 Nov 2024 11:44:22 +0530 Subject: [PATCH 195/311] #OBS-I330 : changed type from hudi to datalake --- api-service/src/services/DatasetService.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 1c864404..206fbb5e 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -344,8 +344,8 @@ class DatasetService { private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const {created_by, updated_by} = draftDataset; - const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); - const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); + const allFields = await tableGenerator.getAllFields(draftDataset, "datalake"); + const draftDatasource = this.createDraftDatasource(draftDataset, "datalake"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) _.set(draftDatasource, "created_by", created_by); @@ -356,9 +356,9 @@ class DatasetService { private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const {created_by, updated_by} = draftDataset; - const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); - const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const allFields = await tableGenerator.getAllFields(draftDataset, "datalake"); + const draftDatasource = this.createDraftDatasource(draftDataset, "datalake"); + const dsId = _.join([draftDataset.dataset_id, "events", "datalake"], "_") const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) From 96ce68092ef551f895b7bba02d4cf38fcd49a912 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 6 Nov 2024 20:32:33 +0700 Subject: [PATCH 196/311] fix for object-store-connector cron (#269) --- .../helm-charts/spark-connector-cron/templates/cronjob.yaml | 2 +- command-service/src/command/connector_registry.py | 3 ++- command-service/src/service/db_service.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index 7d35b32b..f22dcaef 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -70,7 +70,7 @@ spec: - | # Wait for the Spark pod to be ready SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') - kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master=local[*] --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" {{- end }} {{- with .Values.sidecars }} {{- toYaml . | nindent 12 }} diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index ceb2cbba..358c6eb1 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -287,6 +287,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: def execute_query(self, query, params) -> bool: try: result = self.db_service.execute_upsert(sql=query, params=params) + print(f"Connector Registry | {result} rows affected") return result > 0 # Assuming the result is the number of affected rows except Exception as e: print( @@ -426,7 +427,7 @@ def update_connector_registry(self, _id, ver): f"UPDATE connector_registry SET status = 'Retired', updated_date = now() WHERE connector_id = %s AND status = 'Live' AND version != %s", (_id, ver) ) print( - f"Connector Registry | Updated {result} existing rows with connector_id: {_id} and version: {ver}" + f"Connector Registry | Retired {result} versions for connector_id: {_id} and version: {ver}" ) except Exception as e: print( diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py index ded56b91..275f11c6 100644 --- a/command-service/src/service/db_service.py +++ b/command-service/src/service/db_service.py @@ -6,7 +6,7 @@ from config import Config def reconnect(func: Callable): - + def wrapper(db_connection, *args, **kwargs): tdecorator = retry(wait=wait_exponential(), stop=stop_after_attempt(3)) decorated = tdecorator(func) @@ -60,9 +60,9 @@ def execute_upsert(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor.execute(sql, params) + db_connection.commit() record_count = cursor.rowcount db_connection.close() - # print(f"{record_count} inserted/updated successfully") return record_count # @reconnect From b92ed053cd6a940f6b76c3b70bbd3f5c4cdce5e0 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 11 Nov 2024 11:47:40 +0530 Subject: [PATCH 197/311] #OBS-I330 : fixed merging issue --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index bad6f52e..14579fd2 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -96,7 +96,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => defaultConfigs = _.omit(defaultConfigs, "dataset_config.cache_config.redis_db"); } _.set(draftDataset, "updated_by", updated_by); - _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue, key) => { + _.mergeWith(draftDataset, draftDataset,defaultConfigs, (objValue, srcValue, key) => { if (key === "created_by" || key === "updated_by") { if (objValue !== "SYSTEM") { return objValue; @@ -107,6 +107,8 @@ const readyForPublish = async (dataset: Record, updated_by: any) => return objValue; } }); + console.log(draftDataset,"after merge") + const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { throw { From 68c1839a689d958642890e705592d2579c09c1ba Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 11 Nov 2024 12:04:42 +0530 Subject: [PATCH 198/311] #OBS-I330 : removed comment --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 14579fd2..08d0fc98 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -107,7 +107,6 @@ const readyForPublish = async (dataset: Record, updated_by: any) => return objValue; } }); - console.log(draftDataset,"after merge") const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { From 070452e8db02fd5f80bc22b30e2a68d454c008eb Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 11 Nov 2024 13:14:47 +0530 Subject: [PATCH 199/311] #OBS-I330 : Using dataset v2 export api while publishing --- command-service/src/command/dataset_command.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index 74a0a6fe..befca3a8 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -26,7 +26,7 @@ def __init__( self.http_service = http_service self.config_service_host = self.config.find("config_service.host") self.config_service_port = self.config.find("config_service.port") - self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/datasets/v1/export" + self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/v2/datasets/export" def _get_draft_dataset_record(self, dataset_id): query = f""" @@ -66,11 +66,11 @@ def _check_for_live_record(self, dataset_id): def audit_live_dataset(self, command_payload: CommandPayload, ts: int): dataset_id = command_payload.dataset_id dataset_record, data_version = self._check_for_live_record(dataset_id) - export_dataset = self.http_service.post( - url=self.base_url, - body=json.dumps({"dataset_id": dataset_id}), - headers={"Content-Type": "application/json"}, + url=self.base_url + '/{}'.format(dataset_id) + export_dataset = self.http_service.get( + url=url ) + print(export_dataset) if export_dataset.status == 200: result = json.loads(export_dataset.body) object_ = Object( From db6eac034845ce79bfaeb16ddb1e21195dac6e2c Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 11 Nov 2024 14:42:46 +0530 Subject: [PATCH 200/311] #OBS-I330 : removed print statement --- command-service/src/command/dataset_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index befca3a8..d0433884 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -70,7 +70,6 @@ def audit_live_dataset(self, command_payload: CommandPayload, ts: int): export_dataset = self.http_service.get( url=url ) - print(export_dataset) if export_dataset.status == 200: result = json.loads(export_dataset.body) object_ = Object( From e29622f30e5b6f077b7153ab6a2fcf433be9c8fb Mon Sep 17 00:00:00 2001 From: yashash <126703764+yashashkumar@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:45:35 +0530 Subject: [PATCH 201/311] #OBS-I330 : Using dataset v2 export api while publishing (#271) * #OBS-I330 : Using dataset v2 export api while publishing * #OBS-I330 : removed print statement --- command-service/src/command/dataset_command.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index 74a0a6fe..d0433884 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -26,7 +26,7 @@ def __init__( self.http_service = http_service self.config_service_host = self.config.find("config_service.host") self.config_service_port = self.config.find("config_service.port") - self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/datasets/v1/export" + self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/v2/datasets/export" def _get_draft_dataset_record(self, dataset_id): query = f""" @@ -66,10 +66,9 @@ def _check_for_live_record(self, dataset_id): def audit_live_dataset(self, command_payload: CommandPayload, ts: int): dataset_id = command_payload.dataset_id dataset_record, data_version = self._check_for_live_record(dataset_id) - export_dataset = self.http_service.post( - url=self.base_url, - body=json.dumps({"dataset_id": dataset_id}), - headers={"Content-Type": "application/json"}, + url=self.base_url + '/{}'.format(dataset_id) + export_dataset = self.http_service.get( + url=url ) if export_dataset.status == 200: result = json.loads(export_dataset.body) From ffead49d3fcf4759f8fea5b937ad18ff8aa55e90 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 11 Nov 2024 16:37:43 +0530 Subject: [PATCH 202/311] #OBS-I330 : Replacing - with _ for datasource_ref and adding partition key and primary key and timestamp key in column spec of ingestion spec --- api-service/src/services/TableGenerator.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index ec1db785..fd20afc2 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -162,7 +162,9 @@ class TableGenerator extends BaseTableGenerator { return { dataset: dataset.dataset_id, schema: { - table: datasourceRef, + table: _.includes(datasourceRef, '-') + ? _.replace(datasourceRef, /-/g, '_') + : datasourceRef, partitionColumn: partitionKey, timestampColumn: timestampKey, primaryKey: primaryKey, @@ -201,9 +203,6 @@ class TableGenerator extends BaseTableGenerator { private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { const dataFields = _.cloneDeep(allFields); - _.remove(dataFields, { name: primaryKey }) - _.remove(dataFields, { name: partitionKey }) - _.remove(dataFields, { name: timestampKey }) let index = 1; const transformFields = _.map(dataFields, (field) => { return { From 7dfee909f04b4c12d4c15a453cc7217f99fa87f4 Mon Sep 17 00:00:00 2001 From: yashash <126703764+yashashkumar@users.noreply.github.com> Date: Mon, 11 Nov 2024 19:03:54 +0530 Subject: [PATCH 203/311] Hudi fixes (#272) * #OBS-I330 : Using dataset v2 export api while publishing * #OBS-I330 : removed print statement * #OBS-I330 : Replacing - with _ for datasource_ref and adding partition key and primary key and timestamp key in column spec of ingestion spec --- api-service/src/services/TableGenerator.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index ec1db785..fd20afc2 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -162,7 +162,9 @@ class TableGenerator extends BaseTableGenerator { return { dataset: dataset.dataset_id, schema: { - table: datasourceRef, + table: _.includes(datasourceRef, '-') + ? _.replace(datasourceRef, /-/g, '_') + : datasourceRef, partitionColumn: partitionKey, timestampColumn: timestampKey, primaryKey: primaryKey, @@ -201,9 +203,6 @@ class TableGenerator extends BaseTableGenerator { private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { const dataFields = _.cloneDeep(allFields); - _.remove(dataFields, { name: primaryKey }) - _.remove(dataFields, { name: partitionKey }) - _.remove(dataFields, { name: timestampKey }) let index = 1; const transformFields = _.map(dataFields, (field) => { return { From 89571753535310034822d36b46212c23b1b42043 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Mon, 11 Nov 2024 19:23:18 +0530 Subject: [PATCH 204/311] fix: update connector instance id when inserting to db --- command-service/src/command/db_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 01c7e333..6279102f 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -260,7 +260,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): operations_config = connector_config.operations_config if connector_config.operations_config is not None else {} if connector_config.version == 'v2': params = ( - "{0}-{1}".format(dataset_id, connector_config.id), + connector_config.id, dataset_id, connector_config.connector_id, connector_config.connector_config, From dda2f4a6d2e663a1cbdb916587240b60406ee5f5 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 11 Nov 2024 20:42:51 +0530 Subject: [PATCH 205/311] #OBS-I330 : removed merging defaults to keys_config --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 08d0fc98..91443929 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -91,6 +91,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); + defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config"); if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); defaultConfigs = _.omit(defaultConfigs, "dataset_config.cache_config.redis_db"); From 8de4a12f89b6d6f86cab24cf6040101e61d70d5a Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 12 Nov 2024 11:28:00 +0530 Subject: [PATCH 206/311] #OBS-I330 : omit merging defaults to draft dataset for keys_config --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 91443929..0197d39e 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -91,13 +91,15 @@ const readyForPublish = async (dataset: Record, updated_by: any) => let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); - defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config"); + if (_.get(draftDataset, "dataset_config.keys_config")) { + defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config"); + } if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); defaultConfigs = _.omit(defaultConfigs, "dataset_config.cache_config.redis_db"); } _.set(draftDataset, "updated_by", updated_by); - _.mergeWith(draftDataset, draftDataset,defaultConfigs, (objValue, srcValue, key) => { + _.mergeWith(draftDataset, draftDataset, defaultConfigs, (objValue, srcValue, key) => { if (key === "created_by" || key === "updated_by") { if (objValue !== "SYSTEM") { return objValue; From 2e50acf7986ba2d27dbf7eca6c5925d0b0afb6b0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 12 Nov 2024 20:16:39 +0530 Subject: [PATCH 207/311] #OBS-I335: hudi spec fix --- api-service/src/configs/IngestionConfig.ts | 29 +++++++++++ api-service/src/services/DatasetService.ts | 4 +- api-service/src/services/TableGenerator.ts | 59 ++++++++++++++++++---- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts index 8826b264..cfc4d1cf 100644 --- a/api-service/src/configs/IngestionConfig.ts +++ b/api-service/src/configs/IngestionConfig.ts @@ -54,6 +54,13 @@ export const rawIngestionSpecDefaults = { "type": "text", "expr": "$.obsrv_meta.syncts" }, + "hudiSynctsField": { + "name": "obsrv_meta_syncts", + "arrival_format": "text", + "data_type": "date", + "type": "text", + "expr": "$.obsrv_meta.syncts" + }, "dimensions": [ { "type": "string", @@ -64,6 +71,16 @@ export const rawIngestionSpecDefaults = { "name": "obsrv.meta.source.id" } ], + "hudi_dimensions": [ + { + "type": "string", + "name": "obsrv_meta_source_connector" + }, + { + "type": "string", + "name": "obsrv_meta_source_id" + } + ], "flattenSpec": [ { "type": "path", @@ -75,5 +92,17 @@ export const rawIngestionSpecDefaults = { "expr": "$.obsrv_meta.source.connectorInstance", "name": "obsrv.meta.source.id" } + ], + "hudi_flattenSpec": [ + { + "type": "path", + "expr": "$.obsrv_meta.source.connector", + "name": "obsrv_meta_source_connector" + }, + { + "type": "path", + "expr": "$.obsrv_meta.source.connectorInstance", + "name": "obsrv_meta_source_id" + } ] } \ No newline at end of file diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 1c864404..057f15a5 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -344,7 +344,7 @@ class DatasetService { private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const {created_by, updated_by} = draftDataset; - const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); + const allFields = await tableGenerator.getAllFieldsHudi(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) @@ -356,7 +356,7 @@ class DatasetService { private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const {created_by, updated_by} = draftDataset; - const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); + const allFields = await tableGenerator.getAllFieldsHudi(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index ec1db785..ea4d0dcb 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -15,7 +15,7 @@ class BaseTableGenerator { const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { _.mapKeys(schema, function (value, parentKey) { - const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; + const newKey = (prev) ? type === "druid" ? _.join([prev, parentKey], ".") : _.join([prev, parentKey], "_") : parentKey; const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); switch (value["type"]) { case "object": @@ -24,7 +24,7 @@ class BaseTableGenerator { case "array": if (type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { _.mapKeys(_.get(value, "items.properties"), function (value, childKey) { - const objChildKey = _.join([newKey, childKey], ".") + const objChildKey = type === "druid" ? _.join([newKey, childKey], ".") : _.join([newKey, childKey], "_") properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), { expr: _.join([newExpr, "[*].['", childKey, "']"], ""), name: objChildKey, data_type: "array" })) }) } else { @@ -79,6 +79,38 @@ class BaseTableGenerator { _.remove(dataFields, { is_deleted: true }) // Delete all the excluded fields return dataFields; } + + getAllFieldsHudi = async (dataset: Record, type: string): Promise[]> => { + + const { data_schema, denorm_config, transformations_config } = dataset + let dataFields = this.flattenSchema(data_schema, type); + if (!_.isEmpty(denorm_config.denorm_fields)) { + for (const denormField of denorm_config.denorm_fields) { + const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); + const properties = this.flattenSchema(denormDataset.data_schema, type); + const transformProps = _.map(properties, (prop) => { + _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], "_")); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + return prop; + }); + dataFields.push(...transformProps); + } + } + if (!_.isEmpty(transformations_config)) { + const transformationFields = _.map(transformations_config, (tf) => ({ + expr: "$." + tf.field_key, + name: tf.field_key, + data_type: tf.transformation_function.datatype, + arrival_format: tf.transformation_function.datatype, + type: tf.transformation_function.datatype + })) + const originalFields = _.differenceBy(dataFields, transformationFields, "name") + dataFields = _.concat(originalFields, transformationFields) + } + dataFields.push(rawIngestionSpecDefaults.hudiSynctsField) + _.remove(dataFields, { is_deleted: true }) // Delete all the excluded fields + return dataFields; + } } class TableGenerator extends BaseTableGenerator { @@ -92,8 +124,8 @@ class TableGenerator extends BaseTableGenerator { "spec": { "dataSchema": { "dataSource": datasourceRef, - "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset), dataset_config.keys_config.partition_key) }, - "timestampSpec": { "column": this.getTimestampKey(dataset), "format": "auto" }, + "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset, "druid"), dataset_config.keys_config.partition_key) }, + "timestampSpec": { "column": this.getTimestampKey(dataset, "druid"), "format": "auto" }, "metricsSpec": [], "granularitySpec": ingestionSpecDefaults.granularitySpec }, @@ -158,7 +190,7 @@ class TableGenerator extends BaseTableGenerator { const primaryKey = this.getPrimaryKey(dataset); const partitionKey = this.getHudiPartitionKey(dataset); - const timestampKey = this.getTimestampKey(dataset); + const timestampKey = this.getTimestampKey(dataset, "lakehouse"); return { dataset: dataset.dataset_id, schema: { @@ -212,7 +244,7 @@ class TableGenerator extends BaseTableGenerator { "index": index++ } }) - _.each(rawIngestionSpecDefaults.dimensions, (field) => { + _.each(rawIngestionSpecDefaults.hudi_dimensions, (field) => { transformFields.push({ "type": field.type, "name": field.name, @@ -279,20 +311,25 @@ class TableGenerator extends BaseTableGenerator { name: field.name } }), - rawIngestionSpecDefaults.flattenSpec + rawIngestionSpecDefaults.hudi_flattenSpec ) } private getPrimaryKey = (dataset: Record): string => { - return dataset.dataset_config.keys_config.data_key; + return _.replace(dataset.dataset_config.keys_config.data_key, '.', '_'); } private getHudiPartitionKey = (dataset: Record): string => { - return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; + const partitionKey = dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; + return _.replace(partitionKey, '.', '_') } - private getTimestampKey = (dataset: Record): string => { - return dataset.dataset_config.keys_config.timestamp_key; + private getTimestampKey = (dataset: Record, type: string): string => { + const timestamp = dataset.dataset_config.keys_config.timestamp_key; + if (type === "druid") { + return timestamp; + } + return _.replace(timestamp, '.', '_'); } } From 885bbf1196ee16b3fcd72314e9cf9ffedb228b69 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 12 Nov 2024 20:21:33 +0530 Subject: [PATCH 208/311] #OBS-I335: hudi spec fix --- api-service/src/services/TableGenerator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index 7efc753e..f0cb00b2 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -190,7 +190,7 @@ class TableGenerator extends BaseTableGenerator { const primaryKey = this.getPrimaryKey(dataset); const partitionKey = this.getHudiPartitionKey(dataset); - const timestampKey = this.getTimestampKey(dataset, "lakehouse"); + const timestampKey = this.getTimestampKey(dataset, "datalake"); return { dataset: dataset.dataset_id, schema: { From 8c601e4efd97b83a163acf8337ca3c31555c5fe1 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 12 Nov 2024 20:25:32 +0530 Subject: [PATCH 209/311] #OBS-I335: linting fix --- api-service/src/services/TableGenerator.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index f0cb00b2..e749df50 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -194,8 +194,8 @@ class TableGenerator extends BaseTableGenerator { return { dataset: dataset.dataset_id, schema: { - table: _.includes(datasourceRef, '-') - ? _.replace(datasourceRef, /-/g, '_') + table: _.includes(datasourceRef, "-") + ? _.replace(datasourceRef, /-/g, "_") : datasourceRef, partitionColumn: partitionKey, timestampColumn: timestampKey, @@ -232,6 +232,7 @@ class TableGenerator extends BaseTableGenerator { return newHudiSpec; } + // eslint-disable-next-line private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { const dataFields = _.cloneDeep(allFields); @@ -315,12 +316,12 @@ class TableGenerator extends BaseTableGenerator { } private getPrimaryKey = (dataset: Record): string => { - return _.replace(dataset.dataset_config.keys_config.data_key, '.', '_'); + return _.replace(dataset.dataset_config.keys_config.data_key, ".", "_"); } private getHudiPartitionKey = (dataset: Record): string => { const partitionKey = dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; - return _.replace(partitionKey, '.', '_') + return _.replace(partitionKey, ".", "_") } private getTimestampKey = (dataset: Record, type: string): string => { @@ -328,7 +329,7 @@ class TableGenerator extends BaseTableGenerator { if (type === "druid") { return timestamp; } - return _.replace(timestamp, '.', '_'); + return _.replace(timestamp, ".", "_"); } } From 3872825b23842968c867cac9050f7f544b93352f Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Tue, 12 Nov 2024 20:32:20 +0530 Subject: [PATCH 210/311] #OBS-I334 - clear transformations on re-upload --- .../DatasetUpdate/DatasetUpdate.ts | 98 ++++++++++++++----- .../DatasetUpdateValidationSchema.json | 2 +- 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index ba426c43..b3af7b0b 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -67,7 +67,9 @@ const datasetUpdate = async (req: Request, res: Response) => { const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { - const cache_config = _.get(datasetModel, ["dataset_config", "cache_config"]) + const currentSchema = _.get(datasetModel, 'data_schema') + const fieldsRemoved = (datasetReq.data_schema) ? getMissingFieldsInNewSchema(datasetReq.data_schema, currentSchema) : [] + const prev_dataset_config = _.get(datasetModel, ["dataset_config"]) const dataset: Record = { version_key: Date.now().toString(), name: datasetReq.name || _.get(datasetModel, ["name"]), @@ -77,45 +79,87 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config if(datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config if(datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema - if(datasetReq.dataset_config) dataset["dataset_config"] = { ...datasetReq.dataset_config, cache_config } - if(datasetReq.transformations_config) - dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config) - if(datasetReq.denorm_config) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config) + if(datasetReq.dataset_config) dataset["dataset_config"] = { ...prev_dataset_config, ...datasetReq.dataset_config } + if(datasetReq.transformations_config || fieldsRemoved.length > 0) + dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config, fieldsRemoved) + if(datasetReq.denorm_config || fieldsRemoved.length > 0) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config, fieldsRemoved) if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data if(datasetReq.type) dataset["type"] = datasetReq.type - return dataset; } -const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any) => { - const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.field_key") - const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") +const getMissingFieldsInNewSchema = (newSchema: any, oldSchema: any) => { + + const getRemovedPropertiesFieldsNested = (oldProperties: Record, newProperties: Record, path: string[] = []): string[] => { + let removedFields: string[] = []; + for (const key in oldProperties) { + const fullPath = [...path, key].join('.'); + if (!(key in newProperties)) { + removedFields.push(fullPath); + } else if (typeof oldProperties[key] === 'object' && typeof newProperties[key] === 'object') { + removedFields = removedFields.concat( + getRemovedPropertiesFieldsNested(oldProperties[key].properties || {}, newProperties[key].properties || {}, [...path, key]) + ); + } + } + + return removedFields; + } + + const getRemovedPropertiesFields = (oldSchema: any, newSchema: any): string[] => { + const oldProperties = oldSchema.properties || {}; + const newProperties = newSchema.properties || {}; + return getRemovedPropertiesFieldsNested(oldProperties, newProperties); + } + + // Example usage + const removedFieldsNested = getRemovedPropertiesFields(oldSchema, newSchema); + return removedFieldsNested +} - return _.unionWith( - addConfigs, - _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key)}), - (a, b) => { - return a.field_key === b.field_key - } - ) +const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any, fieldsRemoved: string[]) => { + + let updatedConfigs = currentConfigs; + if(newConfigs) { + const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.field_key") + const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + + updatedConfigs = _.unionWith( + addConfigs, + _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key)}), + (a, b) => { + return a.field_key === b.field_key + } + ) + } + if(fieldsRemoved.length > 0) { + updatedConfigs = _.reject(updatedConfigs, (config) => { return _.includes(fieldsRemoved, config.field_key)}) + } + return updatedConfigs } -const mergeDenormConfig = (currentConfig: any, newConfig: any) => { +const mergeDenormConfig = (currentConfig: any, newConfig: any, fieldsRemoved: string[]) => { - const removeConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "remove"}), "value.denorm_out_field") - const addConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "upsert"}), "value") + let updatedConfigs = currentConfig.denorm_fields; + if(newConfig) { + const removeConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "remove"}), "value.denorm_out_field") + const addConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "upsert"}), "value") - const denormFields = _.unionWith( - addConfigs, - _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field)}), - (a, b) => { - return a.denorm_out_field === b.denorm_out_field - } - ) + const denormFields = _.unionWith( + addConfigs, + _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field)}), + (a, b) => { + return a.denorm_out_field === b.denorm_out_field + } + ) + } + if(fieldsRemoved.length > 0) { + updatedConfigs = _.reject(currentConfig.denorm_fields, (config) => { return _.includes(fieldsRemoved, config.denorm_key)}) + } return { - denorm_fields: denormFields + denorm_fields: updatedConfigs } } diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index b31fe674..5d655436 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -224,7 +224,7 @@ "additionalProperties": false } }, - "required": ["indexing_config", "keys_config"], + "required": [], "additionalProperties": false }, "transformations_config": { From 02d51ba1be35c33e8bd1af01cb7c446098d541a0 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Tue, 12 Nov 2024 20:41:56 +0530 Subject: [PATCH 211/311] #OBS-I334 - clear keys_config on re-upload of schema file --- .../src/controllers/DatasetUpdate/DatasetUpdate.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index b3af7b0b..fa10ef36 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -87,6 +87,19 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data if(datasetReq.type) dataset["type"] = datasetReq.type + if(fieldsRemoved.length > 0) { + const keys_config = _.get(dataset["dataset_config"] ? dataset["dataset_config"] : prev_dataset_config, ["keys_config"]) + if(keys_config['data_key'] in fieldsRemoved) { + keys_config['data_key'] = ''; + } + if(keys_config['primary_key'] in fieldsRemoved) { + keys_config['primary_key'] = ''; + } + if(keys_config['timestamp_key'] in fieldsRemoved) { + keys_config['timestamp_key'] = ''; + } + _.set(dataset["dataset_config"], 'keys_config', keys_config) + } return dataset; } From 9ce23736a18b5f38fec890b4d65bc3c6381ab405 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 13 Nov 2024 11:21:02 +0530 Subject: [PATCH 212/311] #OBS-I335: dataset update fix --- .../DatasetUpdate/DatasetUpdate.ts | 120 ++++++++++-------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index fa10ef36..89b31445 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -39,11 +39,11 @@ const validateDataset = (dataset: Record | null, req: Request) => { if (dataset) { if (dataset.api_version !== "v2") { throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "NOT_FOUND", 404) - } - if(dataset.version_key !== versionKey) { + } + if (dataset.version_key !== versionKey) { throw obsrvError(datasetId, "DATASET_OUTDATED", "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", "CONFLICT", 409) } - if(!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { + if (!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { throw obsrvError(datasetId, "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", "Dataset cannot be updated as it is not in draft state", "BAD_REQUEST", 400) } } else { @@ -52,53 +52,57 @@ const validateDataset = (dataset: Record | null, req: Request) => { } const datasetUpdate = async (req: Request, res: Response) => { - + await validateRequest(req) const datasetReq = req.body.request; const datasetModel = await datasetService.getDraftDataset(datasetReq.dataset_id) validateDataset(datasetModel, req) - + const draftDataset = mergeDraftDataset(datasetModel, datasetReq); const userID = (req as any)?.userID; - _.set(draftDataset, "updated_by", userID ) + _.set(draftDataset, "updated_by", userID) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { + const cache_config = _.get(datasetModel, ["dataset_config", "cache_config"]) || {} const currentSchema = _.get(datasetModel, 'data_schema') const fieldsRemoved = (datasetReq.data_schema) ? getMissingFieldsInNewSchema(datasetReq.data_schema, currentSchema) : [] - const prev_dataset_config = _.get(datasetModel, ["dataset_config"]) const dataset: Record = { version_key: Date.now().toString(), name: datasetReq.name || _.get(datasetModel, ["name"]), id: _.get(datasetModel, ["id"]) } - if(datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config - if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config - if(datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config - if(datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema - if(datasetReq.dataset_config) dataset["dataset_config"] = { ...prev_dataset_config, ...datasetReq.dataset_config } - if(datasetReq.transformations_config || fieldsRemoved.length > 0) + if (datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config + if (datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config + if (datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config + if (datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema + const updatedDatasetConfig = _.merge(_.get(datasetModel, "dataset_config"), { ...datasetReq.dataset_config, cache_config }) + if (datasetReq.dataset_config) dataset["dataset_config"] = updatedDatasetConfig + if (datasetReq.transformations_config || fieldsRemoved.length > 0) dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config, fieldsRemoved) - if(datasetReq.denorm_config || fieldsRemoved.length > 0) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config, fieldsRemoved) - if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) - if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) - if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data - if(datasetReq.type) dataset["type"] = datasetReq.type - if(fieldsRemoved.length > 0) { - const keys_config = _.get(dataset["dataset_config"] ? dataset["dataset_config"] : prev_dataset_config, ["keys_config"]) - if(keys_config['data_key'] in fieldsRemoved) { + if (datasetReq.denorm_config || fieldsRemoved.length > 0) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config, fieldsRemoved) + if (datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) + if (datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) + if (datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data + if (datasetReq.type) dataset["type"] = datasetReq.type + if (fieldsRemoved.length > 0) { + const keys_config = _.get(dataset, ["dataset_config", "keys_config"]) + if (_.includes(fieldsRemoved, keys_config['data_key'])) { keys_config['data_key'] = ''; } - if(keys_config['primary_key'] in fieldsRemoved) { + if (_.includes(fieldsRemoved, keys_config['primary_key'])) { keys_config['primary_key'] = ''; } - if(keys_config['timestamp_key'] in fieldsRemoved) { + if (_.includes(fieldsRemoved, keys_config['partition_key'])) { + keys_config['partition_key'] = ''; + } + if (_.includes(fieldsRemoved, keys_config['timestamp_key'])) { keys_config['timestamp_key'] = ''; } - _.set(dataset["dataset_config"], 'keys_config', keys_config) + _.set(dataset["dataset_config"], 'keys_config', keys_config) } return dataset; } @@ -111,22 +115,30 @@ const getMissingFieldsInNewSchema = (newSchema: any, oldSchema: any) => { const fullPath = [...path, key].join('.'); if (!(key in newProperties)) { removedFields.push(fullPath); - } else if (typeof oldProperties[key] === 'object' && typeof newProperties[key] === 'object') { + if (typeof oldProperties[key] === 'object' && oldProperties[key].properties) { + removedFields = removedFields.concat(getRemovedPropertiesFieldsNested(oldProperties[key].properties || {}, {}, [...path, key])); + } + } + else if (typeof oldProperties[key] === 'object' && typeof newProperties[key] === 'object') { removedFields = removedFields.concat( - getRemovedPropertiesFieldsNested(oldProperties[key].properties || {}, newProperties[key].properties || {}, [...path, key]) + getRemovedPropertiesFieldsNested( + oldProperties[key].properties || {}, + newProperties[key].properties || {}, + [...path, key] + ) ); } } - return removedFields; - } - + }; + + const getRemovedPropertiesFields = (oldSchema: any, newSchema: any): string[] => { const oldProperties = oldSchema.properties || {}; const newProperties = newSchema.properties || {}; return getRemovedPropertiesFieldsNested(oldProperties, newProperties); } - + // Example usage const removedFieldsNested = getRemovedPropertiesFields(oldSchema, newSchema); return removedFieldsNested @@ -135,20 +147,20 @@ const getMissingFieldsInNewSchema = (newSchema: any, oldSchema: any) => { const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any, fieldsRemoved: string[]) => { let updatedConfigs = currentConfigs; - if(newConfigs) { - const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.field_key") - const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + if (newConfigs) { + const removeConfigs = _.map(_.filter(newConfigs, { action: "remove" }), "value.field_key") + const addConfigs = _.map(_.filter(newConfigs, { action: "upsert" }), "value") updatedConfigs = _.unionWith( addConfigs, - _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key)}), - (a, b) => { + _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key) }), + (a, b) => { return a.field_key === b.field_key - } + } ) } - if(fieldsRemoved.length > 0) { - updatedConfigs = _.reject(updatedConfigs, (config) => { return _.includes(fieldsRemoved, config.field_key)}) + if (fieldsRemoved.length > 0) { + updatedConfigs = _.reject(updatedConfigs, (config) => { return _.includes(fieldsRemoved, config.field_key) }) } return updatedConfigs } @@ -156,20 +168,20 @@ const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any, fields const mergeDenormConfig = (currentConfig: any, newConfig: any, fieldsRemoved: string[]) => { let updatedConfigs = currentConfig.denorm_fields; - if(newConfig) { - const removeConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "remove"}), "value.denorm_out_field") - const addConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "upsert"}), "value") + if (newConfig) { + const removeConfigs = _.map(_.filter(newConfig.denorm_fields, { action: "remove" }), "value.denorm_out_field") + const addConfigs = _.map(_.filter(newConfig.denorm_fields, { action: "upsert" }), "value") - const denormFields = _.unionWith( + updatedConfigs = _.unionWith( addConfigs, - _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field)}), - (a, b) => { + _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field) }), + (a, b) => { return a.denorm_out_field === b.denorm_out_field - } + } ) } - if(fieldsRemoved.length > 0) { - updatedConfigs = _.reject(currentConfig.denorm_fields, (config) => { return _.includes(fieldsRemoved, config.denorm_key)}) + if (fieldsRemoved.length > 0) { + updatedConfigs = _.reject(updatedConfigs, (config) => { return _.includes(fieldsRemoved, config.denorm_key) }) } return { denorm_fields: updatedConfigs @@ -178,8 +190,8 @@ const mergeDenormConfig = (currentConfig: any, newConfig: any, fieldsRemoved: st const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { - const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.connector_id") - const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + const removeConfigs = _.map(_.filter(newConfigs, { action: "remove" }), "value.connector_id") + const addConfigs = _.map(_.filter(newConfigs, { action: "upsert" }), "value") return _.unionWith( _.map(addConfigs, (config) => { @@ -191,17 +203,17 @@ const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { version: config.version } }), - _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id)}), - (a, b) => { + _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id) }), + (a, b) => { return a.connector_id === b.connector_id - } + } ) } const mergeTags = (currentTags: any, newConfigs: any) => { - const tagsToRemove = _.map(_.filter(newConfigs, {action: "remove"}), "value") - const tagsToAdd = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + const tagsToRemove = _.map(_.filter(newConfigs, { action: "remove" }), "value") + const tagsToAdd = _.map(_.filter(newConfigs, { action: "upsert" }), "value") return _.union(_.pullAll(currentTags, tagsToRemove), tagsToAdd) } From a3c3459c11ce0c1d46c92b47e831a0949edbf845 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 13 Nov 2024 11:33:38 +0530 Subject: [PATCH 213/311] #OBS-I335: loop bound issue fix --- api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 89b31445..cf56ac6a 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -147,7 +147,7 @@ const getMissingFieldsInNewSchema = (newSchema: any, oldSchema: any) => { const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any, fieldsRemoved: string[]) => { let updatedConfigs = currentConfigs; - if (newConfigs) { + if (newConfigs && newConfigs.length) { const removeConfigs = _.map(_.filter(newConfigs, { action: "remove" }), "value.field_key") const addConfigs = _.map(_.filter(newConfigs, { action: "upsert" }), "value") @@ -168,7 +168,7 @@ const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any, fields const mergeDenormConfig = (currentConfig: any, newConfig: any, fieldsRemoved: string[]) => { let updatedConfigs = currentConfig.denorm_fields; - if (newConfig) { + if (_.get(newConfig, "denorm_fields")) { const removeConfigs = _.map(_.filter(newConfig.denorm_fields, { action: "remove" }), "value.denorm_out_field") const addConfigs = _.map(_.filter(newConfig.denorm_fields, { action: "upsert" }), "value") From 939893d193946a69cd3b1cecf1649c723022b27c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 13 Nov 2024 12:06:43 +0530 Subject: [PATCH 214/311] #OBS-I335: dataset updated as per feedbacks --- .../DatasetUpdate/DatasetUpdate.ts | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index cf56ac6a..e9fea051 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -67,7 +67,7 @@ const datasetUpdate = async (req: Request, res: Response) => { const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { - const cache_config = _.get(datasetModel, ["dataset_config", "cache_config"]) || {} + const prev_dataset_config = _.get(datasetModel, ["dataset_config"]) const currentSchema = _.get(datasetModel, 'data_schema') const fieldsRemoved = (datasetReq.data_schema) ? getMissingFieldsInNewSchema(datasetReq.data_schema, currentSchema) : [] const dataset: Record = { @@ -79,8 +79,7 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if (datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config if (datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config if (datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema - const updatedDatasetConfig = _.merge(_.get(datasetModel, "dataset_config"), { ...datasetReq.dataset_config, cache_config }) - if (datasetReq.dataset_config) dataset["dataset_config"] = updatedDatasetConfig + if (datasetReq.dataset_config) dataset["dataset_config"] = { ...prev_dataset_config, ...datasetReq.dataset_config } if (datasetReq.transformations_config || fieldsRemoved.length > 0) dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config, fieldsRemoved) if (datasetReq.denorm_config || fieldsRemoved.length > 0) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config, fieldsRemoved) @@ -89,20 +88,29 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if (datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data if (datasetReq.type) dataset["type"] = datasetReq.type if (fieldsRemoved.length > 0) { - const keys_config = _.get(dataset, ["dataset_config", "keys_config"]) + const keys_config = _.get(dataset["dataset_config"] ? dataset["dataset_config"] : prev_dataset_config, ["keys_config"]) + let modified = false; if (_.includes(fieldsRemoved, keys_config['data_key'])) { + modified = true; keys_config['data_key'] = ''; } - if (_.includes(fieldsRemoved, keys_config['primary_key'])) { - keys_config['primary_key'] = ''; - } if (_.includes(fieldsRemoved, keys_config['partition_key'])) { + modified = true; keys_config['partition_key'] = ''; } if (_.includes(fieldsRemoved, keys_config['timestamp_key'])) { + modified = true; keys_config['timestamp_key'] = ''; } - _.set(dataset["dataset_config"], 'keys_config', keys_config) + if (modified) { + if (dataset["dataset_config"]) { + _.set(dataset["dataset_config"], 'keys_config', keys_config) + } else { + const keys_config = _.get(prev_dataset_config, ["keys_config"]) + dataset["dataset_config"] = { ...prev_dataset_config } + _.set(dataset["dataset_config"], 'keys_config', keys_config) + } + } } return dataset; } From 9880b2fc51e3da2c2e693803692cec46b6826845 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 13 Nov 2024 13:33:30 +0530 Subject: [PATCH 215/311] append base64 prefix upon connector register --- .../src/command/connector_registry.py | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 358c6eb1..40edf5f7 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -176,9 +176,9 @@ def load_ui_metadata(self, extract_out_path): def process_metadata(self, rel_path, connector_source) -> RegistryResponse: result = [] tenant = self.metadata.get("metadata", {}).get("tenant", "") - + self.copy_connector_to_runtime(self.metadata['metadata']['runtime'], connector_source) - + if tenant == "multiple": connector_objects = self.metadata["connectors"] for obj in connector_objects: @@ -213,17 +213,17 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) - + subprocess.run(["rm", "-rf", self.extraction_path]) subprocess.run(["rm", "-rf", self.download_path]) - + if not success: return RegistryResponse( status="failure", message=f"Failed to register connector {connector_id}", statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - result.append(registry_meta.to_dict()) + result.append(registry_meta.to_dict()) return RegistryResponse( status="success", connector_info=result, @@ -266,10 +266,10 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) - + subprocess.run(["rm", "-rf", self.extraction_path]) subprocess.run(["rm", "-rf", self.download_path]) - + if not success: return RegistryResponse( status="failure", @@ -325,16 +325,16 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): ui_spec_json = json.dumps(registry_meta.ui_spec) query =f""" INSERT INTO connector_registry ( - id, connector_id, name, type, category, version, description, - technology, runtime, licence, owner, iconurl, status, source_url, + id, connector_id, name, type, category, version, description, + technology, runtime, licence, owner, iconurl, status, source_url, source, ui_spec, created_by, updated_by, created_date, updated_date, live_date ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT ( connector_id, version - ) DO UPDATE SET + ) DO UPDATE SET id = %s, name = %s, type = %s, @@ -396,14 +396,26 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): datetime.now(), ) return query, params - + def load_file_bytes(self, rel_path: str) -> bytes | None: file_path = Path(self.extraction_path) for item in file_path.glob("*/{}".format(rel_path)): try: + prefixes = { + ".svg": "data:image/svg+xml;base64,", + ".jpeg": "data:image/jpeg;base64,", + ".jpg": "data:image/jpg;base64,", + ".gif": "data:image/gif;base64,", + ".webp": "data:image/webp;base64,", + ".ico": "data:image/x-icon;base64," + } + + prefix = prefixes.get(item.suffix, "data:application/octet-stream;base64,") + print(f"Connector Registry | Image Suffix: {item.suffix} Base64 Prefix in Use: {prefix}") + with open(item, 'rb') as file: file_content = file.read() - encoded = base64.b64encode(file_content).decode("ascii") + encoded = (prefix + base64.b64encode(file_content).decode("ascii")).strip() except IsADirectoryError: print( f"Connector Registry | No value for icon URL given at metadata: {rel_path}" @@ -433,11 +445,11 @@ def update_connector_registry(self, _id, ver): print( f"Connector Registry | An error occurred during the execution of Query: {e}" ) - + def copy_connector_to_runtime(self, runtime: str, connector_source: str): if runtime == "spark": return self.copy_connector_to_spark(connector_source) - + def copy_connector_to_spark(self, connector_source: str): print(f"Connector Registry | copying {connector_source} to spark") @@ -485,7 +497,7 @@ def copy_connector_to_spark(self, connector_source: str): message="failed to copy the connector to spark", statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - + if self.metadata['metadata']['technology'] == "python": pip_install_cmd = [ "kubectl", @@ -507,7 +519,7 @@ def copy_connector_to_spark(self, connector_source: str): message="failed to install the requirements on spark", statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - + return RegistryResponse( status="success", message="connector copied to spark successfully", From c01be2a0eebf05f2839d7457cf4d58b9c5651802 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Wed, 13 Nov 2024 14:53:00 +0530 Subject: [PATCH 216/311] #OBS-I334 - Fix the schema update functionality --- .../DatasetCreate/DatasetCreateValidationSchema.json | 4 ++-- .../DatasetStatusTransition/ReadyToPublishSchema.json | 2 +- .../DatasetUpdate/DatasetUpdateValidationSchema.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 10436e42..15039569 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -179,8 +179,8 @@ "default": true } }, - "required": ["$schema", "type", "properties", "additionalProperties"], - "additionalProperties": false + "required": ["type", "properties"], + "additionalProperties": true }, "dataset_config": { "type": "object", diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index eff208cb..d76acbb8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -188,7 +188,7 @@ "type", "properties" ], - "additionalProperties": false + "additionalProperties": true }, "denorm_config": { "type": "object", diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 5d655436..e6ca971e 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -170,8 +170,8 @@ "default": true } }, - "required": ["$schema", "type", "properties", "additionalProperties"], - "additionalProperties": false + "required": ["type", "properties"], + "additionalProperties": true }, "dataset_config": { "type": "object", From a9157d867951a9d50c5d58cd13f27d604f108abb Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 13 Nov 2024 16:38:17 +0530 Subject: [PATCH 217/311] Data mapping fix (#278) --- .../src/services/SchemaGenerateService/SchemaMapping.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api-service/src/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json index 19843cf5..e9fe2a6f 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaMapping.json +++ b/api-service/src/services/SchemaGenerateService/SchemaMapping.json @@ -66,10 +66,6 @@ "epoch":{ "jsonSchema": "integer", "datasource": "long" - }, - "number": { - "jsonSchema": "number", - "datasource": "double" } } }, From 40b33f2c1c24dca633b7984857e3150ed379d3a6 Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Wed, 13 Nov 2024 21:30:14 +0530 Subject: [PATCH 218/311] #OBS-I335: hudi spec fix (#279) --- api-service/src/services/TableGenerator.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index e749df50..7030ad95 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -15,7 +15,7 @@ class BaseTableGenerator { const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { _.mapKeys(schema, function (value, parentKey) { - const newKey = (prev) ? type === "druid" ? _.join([prev, parentKey], ".") : _.join([prev, parentKey], "_") : parentKey; + const newKey = (prev) ? type === "druid" ? _.join([prev, parentKey], ".") : _.replace(_.join([prev, parentKey], "_"), /\./g, "_") : parentKey; const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); switch (value["type"]) { case "object": @@ -24,7 +24,7 @@ class BaseTableGenerator { case "array": if (type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { _.mapKeys(_.get(value, "items.properties"), function (value, childKey) { - const objChildKey = type === "druid" ? _.join([newKey, childKey], ".") : _.join([newKey, childKey], "_") + const objChildKey = type === "druid" ? _.join([newKey, childKey], ".") : _.replace(_.join([prev, childKey], "_"), /\./g, "_") properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), { expr: _.join([newExpr, "[*].['", childKey, "']"], ""), name: objChildKey, data_type: "array" })) }) } else { @@ -89,7 +89,7 @@ class BaseTableGenerator { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { - _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], "_")); + _.set(prop, "name", _.join([_.replace(denormField.denorm_out_field, /\./g, "_"), prop.name], "_")); _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); return prop; }); @@ -99,7 +99,7 @@ class BaseTableGenerator { if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ expr: "$." + tf.field_key, - name: tf.field_key, + name: _.replace(tf.field_key, /\./g, "_"), data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, type: tf.transformation_function.datatype @@ -316,12 +316,12 @@ class TableGenerator extends BaseTableGenerator { } private getPrimaryKey = (dataset: Record): string => { - return _.replace(dataset.dataset_config.keys_config.data_key, ".", "_"); + return _.replace(dataset.dataset_config.keys_config.data_key, /\./g, "_"); } private getHudiPartitionKey = (dataset: Record): string => { const partitionKey = dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; - return _.replace(partitionKey, ".", "_") + return _.replace(partitionKey, /\./g, "_") } private getTimestampKey = (dataset: Record, type: string): string => { @@ -329,7 +329,7 @@ class TableGenerator extends BaseTableGenerator { if (type === "druid") { return timestamp; } - return _.replace(timestamp, ".", "_"); + return _.replace(timestamp, /\./g, "_"); } } From 8d5fb8ada7f443020f8b595aa1954de5385a2bfd Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 19 Nov 2024 15:02:37 +0530 Subject: [PATCH 219/311] #OBS-I352 : updated swagger doc --- api-service/swagger-doc/openapi_v2.yml | 11395 +++++++++++++++++------ 1 file changed, 8422 insertions(+), 2973 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index d4cd25b5..21dc9e3e 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -1,36 +1,38 @@ openapi: 3.0.0 info: - title: Obsrv API V2 + title: Obsrv v2 apis description: >- Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets. version: 1.0.0 servers: - - url: ":3000" + - url: http://localhost:3000 + - url: http://{{host_ip}} tags: - - name: Dataset CRUD APIs + - name: Dataset api's description: >- The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria. + - name: Connector api's - name: Data Ingest - name: Data query - - name: Data exhaust - name: Query Templates - - name: Dataset copy - - name: Schema validator + - name: Alert Notification-channels + - name: Alert silence + - name: Alerts Wrapper + - name: Alert Metric_alias paths: /v2/datasets/create: post: tags: - - Dataset CRUD APIs + - Dataset api's summary: Dataset create description: >- This API allows you to create new datasets used by the analytical data source. requestBody: - required: true content: application/json: schema: @@ -42,8 +44,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - type: dataset + dataset_id: telemetry_record-t4 + type: event name: sb-telemetry validation_config: validate: true @@ -78,148 +80,418 @@ paths: denorm_fields: - denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry transformations_config: - - field_key: eid + - field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - dataset_config: - data_key: mid - timestamp_key: ets - files_upload_path: ["telemetry.json"] tags: - tag1 - - tag2 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '287' + ETag: + schema: + type: string + example: W/"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ" + Date: + schema: + type: string + example: Mon, 15 Jul 2024 13:14:09 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset Success + summary: 'Success: Dataset created successfullly' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-15T18:44:08+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 276c042c-0f23-4b26-9b10-6fe48bbc2d3a responseCode: OK result: - id: sb-telemetry.1 - version_key: '1713442037275' + id: telemetry_record-t4 + version_key: '1721049248930' example-1: - summary: Master Dataset Success + summary: 'Success: Master dataset created successfully' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:36:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 845076be-d9e7-4246-bb8e-07ae0ce59d1e + responseCode: OK + result: + id: telemetry_record-master + version_key: '1721099200603' + example-2: + summary: 'Success: Minimal request body' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T18:14:59+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9e207f4f-2be6-4a45-ab78-213bea272ae0 + responseCode: OK + result: + id: telemetry_events + version_key: '1721133899306' + example-3: + summary: 'Success: Dataset created successfully with all fields' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-17T18:19:53+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 505fb3bc-ae32-4f5b-a931-adec4d1d84ba responseCode: OK result: - id: sb-telemetry-master.1 - version_key: '1713442037275' + id: telemetry_record-t41 + version_key: '1721220593027' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '362' + ETag: + schema: + type: string + example: W/"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:09:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:39:00+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a07de860-dcbc-4ff6-822e-34b47635c8a3 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '337' + ETag: + schema: + type: string + example: W/"151-a7dJ9XBUyT3AXNxl1TPcraxMX08" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:07:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Dataset contains duplicate denorm out field' + summary: 'Failure: Master dataset already exists' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:59:06+05:30' + ts: '2024-07-16T08:37:28+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 54c5b692-dc37-432e-b556-7f396d7c9e07 - responseCode: BAD_REQUEST + resmsgid: 138b796b-1b68-481a-a59d-1cb695c1adc9 + responseCode: CONFLICT + result: {} error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Duplicate denorm key found - trace: '' + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-master example-1: - summary: 'Failure: Invalid request payload provided' + summary: 'Failure: Dataset already exists' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T18:00:34+05:30' + ts: '2024-07-16T08:38:05+05:30' params: status: FAILED - resmsgid: 615c1e4c-8c19-44fd-b29c-c235e7cbb5f0 - responseCode: BAD_REQUEST + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf62693c-3aa4-42ce-a5ea-4bde340740f5 + responseCode: CONFLICT + result: {} error: - code: DATASET_INVALID_INPUT - message: >- - #additionalProperties should NOT have additional - properties - trace: '' - '409': - description: Conflict + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-t4 + /v2/files/generate-url: + post: + tags: + - Dataset api's + summary: File generate url + description: This API generates presigned URLs to upload or download files from cloud + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.files.generate-url + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '1344' + ETag: + schema: + type: string + example: W/"540-790rZel+H/rDwgvZRxvlUmZ8Gpc" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 02:56:19 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T17:56:06+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 76612ad1-241b-4ce3-8af4-88db860697f4 - responseCode: CONFLICT - error: - code: DATASET_EXISTS - message: Dataset already exists - trace: '' - '500': - description: Internal Server Error + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '355' + ETag: + schema: + type: string + example: W/"163-9oQYJJEaBH3mJAnzDHXn2MxE848" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:03:04 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T18:02:44+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dd1c0e11-fb4c-484c-81fe-82c9e2eee053 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_CREATION_FAILURE - message: Failed to create dataset - trace: '' + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' /v2/datasets/update: patch: tags: - - Dataset CRUD APIs + - Dataset api's summary: Dataset update description: >- This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or - update transformations + update transformations and connectors requestBody: - required: true content: application/json: schema: @@ -231,8 +503,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - version_key: '1717073955640' + dataset_id: telemetry_record-t4 + version_key: '1721135455988' name: sb-telemetry validation_config: validate: true @@ -242,279 +514,254 @@ paths: extraction_key: events dedup_config: drop_duplicates: true - dedup_key: id + dedup_key: ipid dedup_config: drop_duplicates: true - dedup_key: eid + dedup_key: mid data_schema: $schema: https://json-schema.org/draft/2020-12/schema type: object properties: - eid: + midpid: type: string - ets: + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: type: string - required: - - eid + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string additionalProperties: true denorm_config: denorm_fields: - - values: + - value: denorm_key: eid denorm_out_field: userdata - dataset_id: master-telemetry action: remove - - values: + - value: denorm_key: eid denorm_out_field: edata - dataset_id: trip-record - action: add - transformation_config: - - values: - field_key: eid + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert + - value: + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - action: add - dataset_config: - data_key: eid - timestamp_key: ets - tags: - - values: - - tag1 - - tag2 action: remove - - values: - - tag3 - - tag4 - action: add + tags: [] + connectors_config: + - value: + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice + action: upsert + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '331' + ETag: + schema: + type: string + example: W/"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 13:00:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T00:16:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 4b2c0c22-f765-46f9-80af-33d0442db5ca - responseCode: OK - result: - message: Dataset is updated successfully - id: sb-telemetry.1 - version_key: '1713465973004' + examples: + example-0: + summary: 'Success: Minimal dataset update' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:30:45+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 354f1fec-0c39-42ee-a52a-49552f847c11 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134845559' + example-1: + summary: 'Success: Updated successfully' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:27:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 6d835f07-aed5-4e8b-81c2-2142cfb55c52 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134675878' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '369' + ETag: + schema: + type: string + example: W/"171-iNJoyWUecOEsXbHZwx6rld3Sr1I" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:59:21 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Invalid payload provided' + summary: 'Failure: Invalid request' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:22:12+05:30' + ts: '2024-07-16T18:29:21+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9408e137-ada8-48b8-99e9-90d5cdd35e35 + resmsgid: 7d31672b-e5c3-4a6d-afac-d9d78011bcde responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_INPUT_INVALID message: >- - #properties/request/properties/name/type should be - string - trace: '' + #properties/request/required must have required property + 'dataset_id' example-1: - summary: 'Failure: No field provided along with dataset_id' + summary: 'Failure: No fields are provided to update' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:23:06+05:30' + ts: '2024-07-16T18:32:44+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 347f465f-f100-4095-9801-38c4943380c0 + resmsgid: bf99b1e1-7694-4be0-ba5d-e347764736de responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_NO_FIELDS message: >- Provide atleast one field in addition to the dataset_id to update the dataset - trace: '' - example-2: - summary: 'Failure: Cannot update as dataset not in draft state' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:25:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: a9dfc926-c893-4ec3-82b8-bc8601d928a9 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE - message: Dataset cannot be updated as it is not in draft state - trace: '' - example-3: - summary: 'Failure: Dataset contains duplicate denorm out field' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:29:25+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 865b9bef-9b6d-467c-9065-6689126b0f42 - responseCode: BAD_REQUEST - error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Dataset contains duplicate denorm out keys:[userdata] - trace: '' - example-4: - summary: 'Failure: Dataset tags to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:32:05+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9e9ff0bc-d660-4c4e-a2b0-cb11246a6961 - responseCode: BAD_REQUEST - error: - code: DATASET_TAGS_EXISTS - message: Dataset tags already exist - trace: '' - example-5: - summary: 'Failure: Dataset transformations to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:34:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e0a5cc26-032e-4be8-81ef-c7e76867c9c1 - responseCode: BAD_REQUEST - error: - code: DATASET_TRANSFORMATIONS_EXIST - message: Dataset transformations already exists - trace: '' - example-6: - summary: 'Failure: Dataset denorm fields to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:38:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b807580-a2b5-4a44-adc4-80b35b4b6529 - responseCode: BAD_REQUEST - error: - code: DATASET_DENORM_EXISTS - message: Denorm fields already exist - trace: '' '404': description: Not Found - content: - application/json: + headers: + X-Powered-By: schema: - type: object - examples: - example-0: - summary: 'Failure: Dataset does not exists to update' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:20:39+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ebcf0a62-7c95-40f5-adf2-87ab90a40a80 - responseCode: NOT_FOUND - error: - code: DATASET_NOT_EXISTS - message: Dataset does not exists to update - trace: '' - example-1: - summary: 'Failure: Dataset tags to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:33:33+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: c29e7d0f-3eaf-4972-8804-c172db7b6f13 - responseCode: NOT_FOUND - error: - code: DATASET_TAGS_DO_NOT_EXIST - message: Dataset tags do not exist to remove - trace: '' - example-2: - summary: 'Failure: Dataset transformations to update do not exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:36:04+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b5f515b-1958-4597-b603-b4038d4e5846 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to update - trace: '' - example-3: - summary: 'Failure: Dataset transformations to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:37:20+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 69d32411-4252-4b7f-8afa-6d436928c3d3 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to remove - trace: '' - example-4: - summary: 'Failure: Dataset denorm to remove does not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:40:07+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ce78e393-2b74-46da-a70b-2abd06ad3270 - responseCode: NOT_FOUND - error: - code: DATASET_DENORM_DO_NOT_EXIST - message: Denorm fields do not exist to remove - trace: '' - '409': - description: Conflict + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '340' + ETag: + schema: + type: string + example: W/"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:58:30 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -522,20 +769,47 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-24T12:30:18+05:30' + ts: '2024-07-16T18:28:30+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: fe4a438d-441c-4497-a3ec-a0cee2058b48 - responseCode: CONFLICT + resmsgid: bf64703c-bb6b-41bf-bc1a-c85373efd925 + responseCode: NOT_FOUND + result: {} error: - code: DATASET_OUTDATED - message: >- - The dataset is outdated. Please try to fetch latest changes - of the dataset and perform the updates - trace: '' - '500': - description: Internal Server Error + code: DATASET_NOT_EXISTS + message: Dataset does not exists with id:telemetry_record-t41 + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '383' + ETag: + schema: + type: string + example: W/"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:53:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -543,578 +817,230 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:41:33+05:30' + ts: '2024-07-16T18:23:16+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 451daa85-9cb8-459a-b681-a256c6e912f0 - responseCode: INTERNAL_SERVER_ERROR + resmsgid: 02fe03f6-c4c4-48f6-9d84-a32cd52f4c13 + responseCode: CONFLICT + result: {} error: - code: DATASET_UPDATE_FAILURE - message: Failed to update dataset - trace: '' + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates /v2/datasets/read/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset get: tags: - - Dataset CRUD APIs - summary: Dataset read + - Dataset api's + summary: Read dataset description: >- This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status - "Live". - This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, - it creates one using the live dataset and returns the dataset details. -
-
- Valid fields that user can request are - "dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date" + "Live". This API accepts the parameter mode=edit to read the draft + dataset. If a draft dataset is not found, it creates one using the live + dataset and returns the dataset details. parameters: - - name: dataset_id - example: sb_telemetry - in: path - required: true - schema: - type: string - - name: status - example: Draft - in: query - required: false - schema: - type: string - - name: fields - example: name - in: query - required: false + - name: Cookie + in: header schema: type: string + example: >- + connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4 - name: mode - example: edit in: query - required: false schema: type: string + example: edit responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '501' + ETag: + schema: + type: string + example: W/"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:08:55 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Read Draft dataset + summary: 'Success: Read live dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:43:58+05:30' + ts: '2024-07-17T17:38:55+05:30' params: status: SUCCESS - resmsgid: d02c643a-d51d-4c67-8a42-9f43e61f459e + resmsgid: 8c8a2852-54bc-43fb-b063-7f359d11930a responseCode: OK result: - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets - entry_topic: local.ingest + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} + redis_db: 54 example-1: - summary: Read Live dataset + summary: 'Success: Read draft dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:44:54+05:30' + ts: '2024-07-17T17:41:00+05:30' params: status: SUCCESS - resmsgid: 3d0efbc5-dea8-4815-a749-b9f9ae16ccfe + resmsgid: 96fd4f42-fa84-4730-bc79-d241a4e335a1 responseCode: OK result: - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts - exclude_fields: [] - entry_topic: local.ingest + timestamp_key: ets + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - version: 1 + file_upload_path: [] + exclude_fields: [] example-2: - summary: Read specific fields from the dataset + summary: 'Success: Read specific column' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:45:44+05:30' + ts: '2024-07-17T17:42:16+05:30' params: status: SUCCESS - resmsgid: da752dfe-0d88-4dd6-a6a8-d858f5960f7e + resmsgid: 02a6b03a-8bf3-4e37-8dcd-859d3e8f904e responseCode: OK result: - name: sb-telemetry - type: dataset - id: sb-telemetry.1 + name: master-test + type: master + id: master-test example-3: - summary: Read version_key from the dataset + summary: 'Success: Read version_key' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:47:48+05:30' + ts: '2024-07-17T17:45:37+05:30' params: status: SUCCESS - resmsgid: a3ad1e17-671f-4294-a834-2ee3f255c9b3 + resmsgid: 805e848a-d260-47c3-b55c-fc9b8323719e responseCode: OK result: - version_key: '1713510527662' + version_key: '1718791650227' example-4: - summary: 'Read Dataset: Create a draft dataset on edit live' + summary: >- + Success: Read from draft, if not present cerate draft from + live dataset and then read value: id: api.datasets.read ver: v2 - ts: '2024-06-18T14:44:54+05:30' + ts: '2024-07-17T17:49:28+05:30' params: status: SUCCESS - resmsgid: 9f8f6a80-f056-424a-97c4-179e2b9199f2 + resmsgid: da70e25b-6ad0-49a7-a39d-340d1d0c46a7 responseCode: OK result: - id: master-telemetry - dataset_id: master-telemetry - type: master-dataset - name: master-telemetry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - eid: - type: string - arrival_format: text - data_type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - arrival_format: text - data_type: string - updatedBy: - type: string - arrival_format: text - data_type: string - lastAccessedOn: - type: string - arrival_format: text - data_type: string - content_id: - type: string - arrival_format: text - data_type: string - oldLastAccessedOn: - type: string - arrival_format: text - data_type: string - progress: - type: number - arrival_format: number - data_type: number - previousProgress: - type: number - arrival_format: number - data_type: number - contentType: - type: string - arrival_format: text - data_type: string - resourceType: - type: string - arrival_format: text - data_type: string - objectId: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - type: - type: string - arrival_format: text - data_type: string - props: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - ver: - type: string - arrival_format: text - data_type: string - syncts: - type: number - arrival_format: number - data_type: number - '@timestamp': - type: string - arrival_format: text - data_type: string - ets: - type: number - arrival_format: number - data_type: number - context: - type: object - properties: - channel: - type: string - arrival_format: text - data_type: string - pdata: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - env: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - flags: - type: object - properties: - ex_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_validation_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_duplicate_skipped: - type: boolean - arrival_format: boolean - data_type: boolean - arrival_format: object - data_type: object - additionalProperties: true - mid: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - additionalProperties: true - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetry - dataset_config: - data_key: ver - timestamp_key: obsrv_meta.syncts - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 70 - file_upload_path: [] - configurations: - indexConfiguration: - index: - Event Arrival Time: obsrv_meta.syncts - rollupSuggestions: {} - processing: - dedupKeys: [] - dropDuplicates: - - 'Yes' - - 'No' - mergedEvent: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - type: - type: string - required: - - id - - type - eid: - type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - updatedBy: - type: string - lastAccessedOn: - type: string - content_id: - type: string - oldLastAccessedOn: - type: string - progress: - type: number - previousProgress: - type: number - contentType: - type: string - resourceType: - type: string - objectId: - type: string - required: - - oldFirstAccessedOn - - updatedBy - - lastAccessedOn - - content_id - - oldLastAccessedOn - - progress - - previousProgress - - contentType - - resourceType - - objectId - type: - type: string - props: - type: string - required: - - data - - type - - props - ver: - type: string - syncts: - type: number - '@timestamp': - type: string - ets: - type: number - context: - type: object - properties: - channel: - type: string - pdata: - type: object - properties: - id: - type: string - required: - - id - env: - type: string - required: - - channel - - pdata - - env - flags: - type: object - properties: - ex_processed: - type: boolean - pp_validation_processed: - type: boolean - pp_duplicate_skipped: - type: boolean - required: - - ex_processed - - pp_validation_processed - - pp_duplicate_skipped - mid: - type: string - type: - type: string - required: - - actor - - eid - - edata - - ver - - syncts - - '@timestamp' - - ets - - context - - flags - - mid - - type + dataset_id: sample1 + name: sample1 + type: event + status: Live tags: [] - created_by: SYSTEM - updated_by: SYSTEM - version_key: '1718702094143' - version: 1 - status: Draft + version: 2 api_version: v2 - transformations_config: [] + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + data_key: '' + timestamp_key: time + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '307' + ETag: + schema: + type: string + example: W/"133-TQ9WpmutsrDcTNkRRmbWOhUChMk" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:20:17 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1122,19 +1048,48 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:03+05:30' + ts: '2024-07-17T17:50:17+05:30' params: status: FAILED - resmsgid: 302fec39-5070-4464-b02f-56b1d0418147 + resmsgid: bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd responseCode: BAD_REQUEST + result: {} error: code: DATASET_INVALID_FIELDS message: >- The specified fields [newname] in the dataset cannot be found. - trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '304' + ETag: + schema: + type: string + example: W/"130-JL/oBB+GUHTrBp278giBHRvO71I" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:21:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1142,43 +1097,26 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:14+05:30' + ts: '2024-07-17T17:51:12+05:30' params: status: FAILED - resmsgid: cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c + resmsgid: 3aad3842-a76e-4fe8-b635-c7fef5f318f9 responseCode: NOT_FOUND + result: {} error: code: DATASET_NOT_FOUND - message: Dataset with the given dataset_id not found - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.read - ver: v2 - ts: '2024-04-19T12:46:34+05:30' - params: - status: FAILED - resmsgid: 29d82020-101d-4506-8fb8-6f3603057064 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_READ_FAILURE - message: Failed to read dataset - trace: '' + message: >- + Dataset with the given dataset_id:new_telemetry_record.1 not + found /v2/datasets/list: post: tags: - - Dataset CRUD APIs - summary: Dataset List + - Dataset api's + summary: Dataset list description: >- This API allows you to list all datasets. User can apply filters on - dataset status and type, sort the datasets as per requested order. + dataset status and type. requestBody: - required: true content: application/json: schema: @@ -1191,70 +1129,65 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: filters: - type: master-dataset + status: + - Live responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '24340' + ETag: + schema: + type: string + example: W/"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:25:36 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset list success when no filters provided + summary: 'Success: Lists all when no filters provided' value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:54:56+05:30' + ts: '2024-07-17T17:55:36+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e7a2c5c1-5aa9-4422-81a5-6f197d3b689e + resmsgid: 97efe04d-e981-493d-9ee7-a6dad6887d64 responseCode: OK result: data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1264,638 +1197,332 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_dropoff_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 - tags: [] + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_dropoff_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: two.1 - dataset_id: two - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713509698978' - created_date: '2024-04-15T02:22:48.270Z' - updated_date: '2024-04-15T02:24:03.308Z' - transformations_config: - - field_key: eid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - field_key: cid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts - exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid + data_key: '' timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} - count: 6 - example-1: - summary: Dataset list success when status filter provided as array - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:55:39+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 836f853f-9ad8-4567-a9bb-74c5d3bba25d - responseCode: OK - result: - data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts + timestamp_key: date exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: taxt_trip + name: taxt_trip + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: obsrv_meta.syncts exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 - tags: [] + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - count: 2 - example-2: - summary: Dataset list success when status filter provided as string - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:56:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 469ef16b-4211-4b71-b063-bbd81cc9df02 - responseCode: OK - result: - data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: telemetry_record-t4 + name: sb-telemetry + type: event + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_events + name: sb-telemetry + type: event + status: Draft + tags: [] + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_record-master + name: sb-telemetry + type: master + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 dataset_config: - data_key: skill + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: generate-schema + name: generate-schema + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' timestamp_key: '' exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + api_version: null dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1905,1453 +1532,6450 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - count: 2 - example-3: - summary: Dataset list success when type filter provided - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:57:28+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 7d9aeb08-2952-485f-9944-4d118da7842e - responseCode: OK - result: - data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + api_version: null dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_pickup_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 1 - example-4: - summary: Dataset list success based on sortBy values + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-test + name: trip-test + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample-trip-details + name: sample-trip-details + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 30 + example-1: + summary: 'Success: Filter based on status as array' value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:58:27+05:30' + ts: '2024-07-17T17:57:38+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 270a4e50-f7e5-4562-ae75-7a967866d065 + resmsgid: 31aba5bc-8492-45ce-be0e-8c52d8716014 responseCode: OK result: data: - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_dropoff_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill + data_key: userid timestamp_key: '' exclude_fields: [] entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 2 - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - status: FAILED - resmsgid: f9e1fae9-4660-409d-bb4d-e29888983d4b - responseCode: BAD_REQUEST - error: - code: DATASET_LIST_INPUT_INVALID - message: >- - #properties/params/additionalProperties should NOT have - additional properties - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:59:15+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 81c3a4a5-e8ac-41de-be00-6d0300ecb106 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_LIST_FAILURE - message: Failed to list dataset - trace: '' - /v2/datasets/diff/{dataset_id}: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + count: 16 + example-2: + summary: 'Success: Filter based on status as string' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T17:59:18+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a08c7ea0-bb1c-4998-b47d-a76e38e87e31 + responseCode: OK + result: + data: + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 8 + example-3: + summary: 'Success: Filter based on dataset type' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:00:41+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 0d1ff2de-42c9-4192-b75d-84f711dbfb55 + responseCode: OK + result: + data: + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + count: 2 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '312' + ETag: + schema: + type: string + example: W/"138-XQplwhrgIYKIg0qtQdRCYWIGTNM" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:32:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:02:26+05:30' + params: + status: FAILED + resmsgid: add9dbe0-f362-4f99-890c-3387c998a049 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_LIST_INPUT_INVALID + message: >- + #properties/params/required must have required property + 'msgid' + /v2/datasets/dataschema: + post: + tags: + - Dataset api's + summary: Data schema generator + description: This api is used to generate data schema for the given dataset event. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + data: + - eid: IMPRESSION + ets: 1672657002221 + ver: '3.0' + mid: IMPRESSION:2b5834e196f485c17c4e49d292af43c0 + actor: + id: 0c45959486f579c24854d40a225d6161 + type: User + context: + channel: '01268904781886259221' + pdata: + id: staging.diksha.portal + ver: 5.1.0 + pid: sunbird-portal + env: public + sid: 23850c90-8a8c-11ed-95d0-276800e1048c + did: 0c45959486f579c24854d40a225d6161 + cdata: [] + rollup: + l1: '01268904781886259221' + uid: anonymous + object: {} + tags: + - '01268904781886259221' + edata: + type: view + pageid: login + subtype: pageexit + uri: >- + https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4 + visits: [] + syncts: 1672657005814 + '@timestamp': '2023-01-02T10:56:45.814Z' + flags: + ex_processed: true + - eid: IMPRESSION + ets: 1672656997928 + ver: '3.0' + mid: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: POST + - url: /v1/org/search + - duration: 0 + - status: OK + - eid: LOG + ets: 1672656998024 + ver: '3.0' + mid: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + type=system, message=EXIT LOG: method : POST, url: + /v1/org/search , For Operation : orgSearch, + params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + errmsg=Invalid value null for parameter hashTagId. + Please provide a valid value., + resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + err=UOS_ORGSER0017, status=FAILED, + responseCode=400}]}} + params: [] + - eid: LOG + ets: 1672657004961 + ver: '3.0' + mid: f34112c7242a3e3a26f0015796b029c2 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: f34112c7242a3e3a26f0015796b029c2 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + ElasticSearchRestHighImpl:search: calling search for + index org_alias, with query = + {"from":0,"size":250,"query":{"bool":{"must":[{"term":{"isTenant.raw":{"value":true,"boost":1.0}}},{"term":{"slug.raw":{"value":"ntp","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":[],"excludes":[]}} + params: [] + - eid: LOG + ets: 1672657006595 + ver: '3.0' + mid: d23ff123-40f0-4262-a69b-b75b46d315a1 + actor: + id: 930a3994-cbe7-4e84-936f-4974096af6f2 + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: d23ff123-40f0-4262-a69b-b75b46d315a1 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, + type=system, message=ENTRY LOG: method : GET, url: + /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 + , For Operation : getUserRolesById, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006611 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, + type=system, message=ENTRY LOG: method : GET, url: + /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , + For Operation : getUserProfileV5, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006620 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user_roles + WHERE userId=?; + params: [] + - eid: LOG + ets: 1672657006645 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: GET + - url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 + - duration: 0 + - status: OK + - eid: LOG + ets: 1672657007238 + ver: '3.0' + mid: d4d34fde-c407-efb6-03bd-9f892ca0f114 + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.portal + pid: learner-service + ver: 5.0.0 + env: User + did: d904c90d9f81ddac20141b94ddd606a0 + cdata: + - id: d4d34fde-c407-efb6-03bd-9f892ca0f114 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user WHERE + id=?; + params: [] + config: + dataset: financial_transactions + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '10760' + ETag: + schema: + type: string + example: W/"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:02:50 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:32:50+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1309aea0-9a97-46e9-bc5e-a16a8a7fb624 + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' format + type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'sid'. The property sid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.sid + - message: >- + The Property 'context.sid' appears to be + 'uuid' format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.context.properties.sid + arrival_format: text + data_type: string + did: + type: string + arrival_format: text + data_type: string + cdata: + type: array + items: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'context.cdata[*].id' + appears to be 'uuid' format type. + advice: >- + Suggest to not to index the high + cardinal columns + resolutionType: DEDUP + severity: LOW + path: >- + properties.context.properties.cdata.items.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: array + data_type: array + rollup: + type: object + properties: + l1: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at + property: 'l1'. The property l1: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System + has updated the property schema to + optional + resolutionType: OPTIONAL + severity: MEDIUM + path: >- + properties.context.properties.rollup.properties.l1 + arrival_format: text + data_type: string + arrival_format: object + data_type: object + uid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uid'. The property uid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.uid + arrival_format: text + data_type: string + arrival_format: object + data_type: object + object: + type: object + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'object'. The property object: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.object + arrival_format: object + data_type: object + tags: + type: array + items: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'tags'. The property tags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.tags + arrival_format: array + data_type: array + edata: + type: object + properties: + type: + type: string + arrival_format: text + data_type: string + pageid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'pageid'. The property pageid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.pageid + arrival_format: text + data_type: string + subtype: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'subtype'. The property subtype: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.subtype + arrival_format: text + data_type: string + uri: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uri'. The property uri: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.uri + - message: >- + The Property 'edata.uri' appears to be 'uri' + format type. + severity: '' + path: properties.edata.properties.uri + arrival_format: text + data_type: string + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'visits'. The property visits: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.visits + arrival_format: array + data_type: array + level: + type: string + arrival_format: text + data_type: string + message: + type: string + arrival_format: text + data_type: string + params: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + syncts: + type: integer + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'syncts'. The property syncts: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.syncts + - message: >- + The Property 'syncts' appears to be 'epoch' format + type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + '@timestamp': + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + '@timestamp'. The property @timestamp: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.@timestamp + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + flags: + type: object + properties: + ex_processed: + type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'ex_processed'. The property ex_processed: + only 1 time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags.properties.ex_processed + arrival_format: boolean + data_type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'flags'. The property flags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + mid: + path: $.mid + cardinality: 67 + index: false + actor.id: + path: $.actor.properties.id + cardinality: 56 + index: false + context.sid: + path: $.context.properties.sid + cardinality: 11 + index: true + edata.uri: + path: $.edata.properties.uri + cardinality: 11 + index: true + context.cdata[*].id: + path: $.context.properties.cdata.items.properties.id + cardinality: 62 + index: false + processing: + dedupKeys: + - mid + - context.cdata[*].id + - actor.id + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:03:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request body' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:33:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bbcc86c2-042d-4f77-bb6e-e1c9116df570 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'data' + example-1: + summary: 'Failure: Invalid request (config not provided)' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:35:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1f856c5e-37f0-41e9-96fb-642471228da2 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'config' + /v2/datasets/status-transition: + post: + tags: + - Dataset api's + summary: Dataset Status Transition + description: >- + This API allows you to perform status transition between 2 states. + Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to + Live, Live to Retired and even Delete a dataset. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: ReadyToPublish + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete success: Deleted dataset successfully' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T12:18:54+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + responseCode: OK + result: + message: Dataset status transition to Delete successful + dataset_id: trip-data + example-1: + summary: 'Live success: Dataset published successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:21:42+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + responseCode: OK + result: + message: Dataset status transition to Live successful + dataset_id: telemetry_record + example-2: + summary: 'ReadyToPublish success: Dataset is ready to publish' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:30:04+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + responseCode: OK + result: + message: Dataset status transition to ReadyToPublish successful + dataset_id: telemetry-events + example-3: + summary: 'Retire success: Dataset retired successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:22:58+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + responseCode: OK + result: + message: Dataset status transition to Retire successful + dataset_id: sb-telemetry + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Live failure: Dataset in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T15:37:43+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + responseCode: BAD_REQUEST + error: + code: DATASET_LIVE_FAILURE + message: >- + Transition failed for dataset: sb-telemetry2 + status:Draft with status transition to Live + trace: '' + example-1: + summary: 'ReadyToPublish failure: Incomplete dataset configs' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:36:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + responseCode: BAD_REQUEST + error: + code: DATASET_CONFIGS_INVALID + message: >- + #properties/denorm_config/properties/denorm_fields/items/required + must have required property 'dataset_name' + trace: '' + example-2: + summary: 'ReadyToPublish failure: Dataset not in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:38:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + responseCode: BAD_REQUEST + error: + code: DATASET_READYTOPUBLISH_FAILURE + message: >- + Transition failed for dataset: telemetry-events + status:Retired with status transition to ReadyToPublish + trace: '' + example-3: + summary: 'Retire Failure: Dataset is already retired' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:42:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 + responseCode: BAD_REQUEST + error: + code: DATASET_RETIRE_FAILURE + message: >- + Transition failed for dataset: master-telemetrry + status:Retired with status transition to Retire + trace: '' + example-4: + summary: >- + Retire failure: Cannot retire master dataset as it is used + by other datasets + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:01:41+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 + responseCode: BAD_REQUEST + error: + code: DATASET_IN_USE + message: >- + Failed to retire dataset as it is in use. Please retire + or delete dependent datasets before retiring this + dataset + trace: '' + example-5: + summary: 'Failure: Invalid request payload provided' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:03:56+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 + responseCode: BAD_REQUEST + error: + code: DATASET_STATUS_INVALID_INPUT + message: >- + #properties/request/properties/status/enum should be + equal to one of the allowed values + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete failure: Dataset not found to delete' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:25:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: master' + trace: '' + example-1: + summary: 'Live failure: Dataset not found to publish' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:35:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: telemetry-data' + trace: '' + example-2: + summary: 'Retire Failure: Dataset not found to retire' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:40:31+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: sb-telemetry2' + trace: '' + /v2/datasets/import: + post: + tags: + - Dataset api's + summary: Dataset import + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + id: sb-telemetry_draft_data + dataset_id: sb-telemetryRPF_draft_data + name: sb-telemetry_draft_data + type: event + extraction_config: + is_batch_event: false + dedup_config: + drop_duplicates: false + dedup_key: id + dedup_period: 604800 + extraction_key: events + validation_config: + validate: true + mode: Strict + dedup_config: + dedup_key: id + drop_duplicates: true + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + syncts: + type: integer + arrival_format: number + data_type: integer + ets: + type: integer + arrival_format: number + data_type: epoch + suggestions: + - message: >- + The Property 'ets' appears to be 'date-time' + format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.ets + flags: + type: object + properties: + ex_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_validation_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_duplicate_skipped: + type: boolean + arrival_format: boolean + data_type: boolean + user_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + device_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + loc_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + content_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + coll_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + arrival_format: object + data_type: object + additionalProperties: false + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + edata: + type: object + properties: + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + type: + type: string + arrival_format: text + data_type: string + duration: + type: number + arrival_format: number + data_type: number + size: + type: integer + arrival_format: number + data_type: integer + query: + type: string + arrival_format: text + data_type: string + filters: + type: object + properties: + objectType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + version: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + status: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + id: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + isRootOrg: + type: boolean + arrival_format: boolean + data_type: boolean + trackable.enabled: + type: string + arrival_format: text + data_type: string + channel: + type: object + arrival_format: object + data_type: object + additionalProperties: false + framework: + type: object + arrival_format: object + data_type: object + additionalProperties: false + resourceType: + type: object + arrival_format: object + data_type: object + additionalProperties: false + identifier: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + contentType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + mimeType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + hashTagId: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + compatibilityLevel: + type: object + properties: + min: + type: integer + arrival_format: number + data_type: integer + max: + type: integer + arrival_format: number + data_type: integer + arrival_format: object + data_type: object + additionalProperties: false + createdBy: + type: string + arrival_format: text + data_type: string + mediaType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + origin: + type: string + arrival_format: text + data_type: string + primaryCategory: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + trackable: + enabled: + type: string + arrival_format: object + data_type: object + additionalProperties: false + sort: + type: object + properties: + lastUpdatedOn: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + topn: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + pageid: + type: string + arrival_format: text + data_type: string + uri: + type: string + arrival_format: text + data_type: string + subtype: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + data: + type: string + arrival_format: text + data_type: string + uaspec: + type: object + properties: + agent: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + system: + type: string + arrival_format: text + data_type: string + platform: + type: string + arrival_format: text + data_type: string + raw: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + state: + type: string + arrival_format: text + data_type: string + props: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + prevstate: + type: string + arrival_format: text + data_type: string + dspec: + type: object + properties: + os: + type: string + arrival_format: text + data_type: string + make: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + idisk: + type: number + arrival_format: number + data_type: number + edisk: + type: number + arrival_format: number + data_type: number + scrn: + type: number + arrival_format: number + data_type: number + camera: + type: string + arrival_format: text + data_type: string + cpu: + type: string + arrival_format: text + data_type: string + sims: + type: integer + arrival_format: number + data_type: integer + webview: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + extra: + type: object + properties: + pos: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + values: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + query: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + mode: + type: string + arrival_format: text + data_type: string + duration: + type: string + arrival_format: object + data_type: object + additionalProperties: false + '@timestamp': + type: string + suggestions: + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + l2: + type: string + arrival_format: text + data_type: string + l3: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + did: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + object: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + version: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + tags: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + additionalProperties: false + denorm_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + denorm_fields: + - denorm_key: actor.id + dataset_id: master-dataset + denorm_out_field: userdata + router_config: + topic: sb-telemetry + dataset_config: + keys_config: + timestamp_key: obsrv_meta.syncts + data_key: '' + partition_key: '' + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + cache_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + redis_db: 0 + file_upload_path: [] + transformations_config: [] + connectors_config: [] + tags: + - tag1 + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1724333643940' + api_version: v2 + sample_data: {} + entry_topic: dev.ingest + created_date: '2024-07-23T18:35:15.690Z' + updated_date: '2024-08-22T13:34:08.041Z' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: overwrite + in: query + schema: + type: boolean + example: 'false' + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '346' + ETag: + schema: + type: string + example: W/"15a-SX5o6plRdfy+akEYhbMwq9zl+oU" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:34:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Fully imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:04:26+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: fd87393e-4b1b-4cf0-8e9a-a0529d691cc3 + responseCode: OK + result: + message: Dataset is imported successfully + data: + id: sb-telemetry_draft_data + version_key: '1732001666671' + example-1: + summary: 'Success: Partially imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T12:59:41+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 379ced0b-eb06-445f-a7e3-8737cb16c351 + responseCode: OK + result: + message: Dataset is partially imported + data: + id: sb-telemetry_draft_data + version_key: '1732001381825' + ignoredFields: + denorm_fields: + - config: + denorm_key: actor.id + denorm_out_field: userdata + dataset_id: master-dataset + details: Master dataset does not exist + example-2: + summary: 'Success: Imported new dataset [overwrite=false]' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:05:32+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1a50e1b1-14b0-4178-aba4-9e09a63d8d1d + responseCode: OK + result: + message: Dataset is imported successfully + data: + id: sb-telemetry_draft + version_key: '1732001732045' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-Dxem7prJM89DOqth+bdZbh0UI1g" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:38:05 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:08:05+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 244318a5-d933-41a5-add6-14f299281069 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_IMPORT_INVALID_CONFIGS + message: >- + #properties/request/required must have required property + 'id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '351' + ETag: + schema: + type: string + example: W/"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:31:11 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:01:11+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1e5b34cd-4473-4439-8cc5-5ff917f47e48 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: >- + Dataset with dataset_id: sb-telemetryRPF_draft_data already + exists. + /v2/datasets/export/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset get: tags: - - Dataset CRUD APIs - summary: Dataset Diff - description: >- - This API allows you to return the difference between the live and draft - dataset by returning additions, deletions and modifications in the - dataset. + - Dataset api's + summary: Dataset export + parameters: + - name: status + in: query + schema: + type: string + example: Live + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '5155' + ETag: + schema: + type: string + example: W/"1423-Tx46QCrIX8So3vSGVkd865IYeEY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:16:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Live Dataset export success' + value: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:46:01+05:30' + params: + status: SUCCESS + resmsgid: ae57c620-2858-40f9-9d8e-33f8f259449d + responseCode: OK + result: + id: content-data-v2 + dataset_id: content-data-v2 + type: master + name: content-data-v2 + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + email: + key: email + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mobile: + key: mobile + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + fare_details: + key: fare_details + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + fare_amount: + key: fare_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + extra: + key: extra + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mta_tax: + key: mta_tax + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tolls_amount: + key: tolls_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + improvement_surcharge: + key: improvement_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + total_amount: + key: total_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + congestion_surcharge: + key: congestion_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + additionalProperties: true + dedup_config: + drop_duplicates: true + dedup_key: tripID + dedup_period: 604800 + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: content-data-v2 + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-2_addedf.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + timestamp_key: tpep_pickup_datetime + data_key: '' + partition_key: '' + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 7 + tags: [] + status: Live + created_by: SYSTEM + updated_by: SYSTEM + data_version: 1 + api_version: v2 + version: 1 + sample_data: + mergedEvent: + tripID: ccbb970c-4d10-4247-be0c-a480996f3429 + VendorID: '1' + tpep_pickup_datetime: '2023-02-26 00:36:10' + tpep_dropoff_datetime: '2023-09-15 00:45:45' + passenger_count: '2' + trip_distance: '1.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '263' + payment_type: '1' + primary_passenger: + email: Jacey_Hintz@yahoo.com + mobile: 247-492-3370 + fare_details: + fare_amount: '8.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.45' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '12.25' + congestion_surcharge: '' + entry_topic: local.ingest + created_date: '2024-10-14T06:23:44.588Z' + updated_date: '2024-10-14T06:23:44.588Z' + example-1: + summary: 'Success: ReadyToPublish Dataset exported successfully' + value: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:48:11+05:30' + params: + status: SUCCESS + resmsgid: 222847a0-32bf-4c63-be68-76f0e51258af + responseCode: OK + result: + id: mydataset + dataset_id: mydataset + name: mydataset + type: event + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + validation_config: + validate: true + mode: Strict + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + email: + key: email + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mobile: + key: mobile + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + additionalProperties: false + fare_details: + key: fare_details + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + fare_amount: + key: fare_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + extra: + key: extra + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + mta_tax: + key: mta_tax + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + tolls_amount: + key: tolls_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + improvement_surcharge: + key: improvement_surcharge + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + total_amount: + key: total_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + congestion_surcharge: + key: congestion_surcharge + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + additionalProperties: false + additionalProperties: false + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: mydataset + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-1 (4) + (1)_5ec855.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 + transformations_config: [] + connectors_config: [] + tags: [] + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1731420131072' + api_version: v2 + sample_data: + mergedEvent: + tripID: 4c77e9d5-538d-4eb7-8db1-4c2c32860aa8 + VendorID: '2' + tpep_pickup_datetime: '2023-11-02 00:18:42' + tpep_dropoff_datetime: '2023-11-02 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Willa67@gmail.com + mobile: 1-720-981-6399 x77055 + fare_details: + fare_amount: 7 + extra: 0.5 + mta_tax: 0.5 + tip_amount: 0 + tolls_amount: 0 + improvement_surcharge: 0.3 + total_amount: 8.3 + congestion_surcharge: 0 + entry_topic: local.ingest + created_date: '2024-11-11T21:31:21.390Z' + updated_date: '2024-11-12T08:32:27.931Z' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '291' + ETag: + schema: + type: string + example: W/"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:20:43 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:50:43+05:30' + params: + status: FAILED + resmsgid: 2097228c-7872-4217-a06a-0ec08e3dc67d + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_EXPORT_FAILURE + message: Dataset with status:Draft cannot be exported + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:15:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:45:10+05:30' + params: + status: FAILED + resmsgid: fd6fdae2-a002-4c15-81e2-3cce539a2a5a + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_FOUND + message: >- + Dataset with the given dataset_id:v1-copy not found to + export + /v2/datasets/copy: + post: + tags: + - Dataset api's + summary: Dataset copy + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + source: + datasetId: dataset-telemetry + isLive: true + destination: + datasetId: bew-copy-live2 parameters: - - name: dataset_id - example: telemetry - in: path - required: true + - name: Content-Type + in: header schema: type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '296' + ETag: + schema: + type: string + example: W/"128-bi4wSfgkyMP13qFE8VPTmgDdgLA" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:26:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - examples: - example-0: - summary: Data Format Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T16:55:25+05:30' - params: - status: SUCCESS - resmsgid: c58aa5f7-f182-4dff-9ee9-16f508e53c51 - responseCode: OK - result: - additions: - - type: dataFormat - items: - - name: extraction_key - value: mid - - name: batch_id - value: id - - name: dedup_config.dedup_key - value: id - modifications: - - type: dataFormat - items: - - name: is_batch_event - value: - from: false - to: true - deletions: [] - example-1: - summary: Timestamp and data key diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:00:44+05:30' - params: - status: SUCCESS - resmsgid: 8ec5fa57-5105-4793-ae96-3a7e0550dbe9 - responseCode: OK - result: - additions: [] - modifications: - - type: timestamp - value: - from: syncts - to: ets - - type: dataKey - value: - from: eid - to: mid - deletions: [] - example-2: - summary: Data Schema Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-28T15:40:48+05:30' - params: - status: SUCCESS - resmsgid: c364416d-b903-4af6-a39b-a4e52c4bf6b5 - responseCode: OK - result: - additions: - - type: dataSchema - items: - - name: eid_1 - value: - arrivalFormat: boolean - dataType: boolean - absolutePath: $.properties.eid_1 - isRequired: false - modifications: - - type: dataSchema - items: - - field: eid - name: type - value: - from: string - to: boolean - - field: eid - name: arrivalFormat - value: - from: text - to: boolean - - field: eid - name: dataType - value: - from: string - to: boolean - deletions: - - type: dataSchema - items: - - name: ver - example-3: - summary: Data Validation Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:05:28+05:30' - params: - status: SUCCESS - resmsgid: ad6b4977-c341-4193-a42f-9b45993e5f8e - responseCode: OK - result: - additions: [] - modifications: - - type: validation - items: - - field: mode - value: - from: IgnoreNewFields - to: Strict - deletions: [] - example-4: - summary: Dataset Dedup diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:07:58+05:30' - params: - status: SUCCESS - resmsgid: e59b841b-e35f-462d-adb1-31018d718f6a - responseCode: OK - result: - additions: - - type: dedup - items: - - name: dedup_key - value: mid - modifications: - - type: dedup - items: - - name: drop_duplicates - value: - from: false - to: true - deletions: [] - example-5: - summary: Dataset Denorm Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:17:35+05:30' - params: - status: SUCCESS - resmsgid: ee558c17-b3a6-41cc-9b07-64105053394e - responseCode: OK - result: - additions: - - type: denorm - items: - - name: syncts - value: - denorm_key: actor.id - dataset_name: mid - modifications: [] - deletions: - - type: denorm - items: - - name: mid - value: - denorm_key: actor.id - dataset_name: mid - example-6: - summary: Dataset transformation diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:25:12+05:30' - params: - status: SUCCESS - resmsgid: f4389891-9c09-4ccd-affe-39970590045b - responseCode: OK - result: - additions: - - type: transformations - items: - - name: skill_id - value: - transformation_function: - type: mask - condition: null - expr: skill_id - mode: Strict - metadata: - section: pii - modifications: - - type: transformations - items: - - name: new - field: transformation_function.type - value: - from: jsonata - to: encrypt - metadata: - section: transformation - deletions: - - type: transformations - items: - - name: mid - value: - metadata: - section: transformation - - name: name - value: - metadata: - section: transformation - - name: neew - value: - metadata: - section: additionalFields - /v2/files/generate-url: + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:56:12+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 28072631-8e80-45eb-906f-d933a90646d0 + responseCode: OK + result: + dataset_id: new-copy-live2 + message: Dataset clone successful + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '328' + ETag: + schema: + type: string + example: W/"148-jUDshghYf/jOhmmVkvtHlfJT864" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:22:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:52:41+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: f2fd51bb-fc18-4278-a229-47fa25398e69 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_EXISTS + message: Dataset dataset-telemetry does not exists + /v2/connectors/list: post: tags: - - Dataset CRUD APIs - summary: Files generate URL - description: >- - This API allows you to generate pre signed urls from cloud providers for - getting and putting files. On access: write, upload urls are generated to put files and on access: read, download urls are generated to get files. + - Connector api's + summary: Connector list requestBody: - content: - application/json: - examples: - example1: - summary: Generate upload URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: write - example2: - summary: Generate download URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: read + content: + application/json: + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Draft responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '250' + ETag: + schema: + type: string + example: W/"fa-+eWKIfUxsWBGuJy23qSucgLXke4" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:55:51 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Success: Generate Put URL' + summary: 'Success: Filtered based on status' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:04:15+05:30' + ts: '2024-07-30T15:25:51+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a79f1260-1b34-4b96-9c12-cf0f5600e9f3 + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a responseCode: OK result: - - filePath: >- - obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=0zWi7R9wqUT6uW0QuN%2F0nyvaloSCqRPTvzHn%2BpkCWdk%3D - - filePath: >- - obsrv-onest/api-service/user_uploads/school_data_14092a.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data_14092a.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=35g%2F46r%2FOI490%2B8pKry9mEbAgXOXznmCyw7L%2FSLITM0%3D + data: [] + count: 0 example-1: - summary: 'Success: Generate Get URL' + summary: 'Success: Filtered based on category' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:05:47+05:30' + ts: '2024-07-31T18:55:03+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 141717fb-9b94-463d-b0f1-1bab40b187cc + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab responseCode: OK result: - - filePath: obsrv-onest/api-service/user_uploads/telemetry.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=UEWKMP9wcETNKA1sXo%2BVI6qZ7Iqk%2BJg6kglaEXz7%2B1Y%3D - - filePath: obsrv-onest/api-service/user_uploads/school_data.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=jXzootTAoUAWLYKF9AYwKh9Mzl%2FVctlebTBHVz%2BbaKM%3D - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Invalid access value requested' + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + count: 4 + example-2: + summary: 'Success: Connectors list with all filter options' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:08:23+05:30' + ts: '2024-07-31T18:56:32+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b1da48a0-5381-49a1-9d5f-ce6c7c65fdcf - responseCode: BAD_REQUEST - error: - code: FILES_GENERATE_URL_INPUT_INVALID - message: >- - #properties/request/properties/access/enum should be - equal to one of the allowed values - trace: '' - example-1: - summary: >- - Failure: Number of file url requested exceeds the limit - configured + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 11a2f537-bd98-405b-97e5-0f0d5b86b2c3 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-3: + summary: 'Success: Connectors list without filters' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:10:29+05:30' + ts: '2024-07-31T18:57:37+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5828abd2-2934-40ef-8fa1-a9bca1e7e24e - responseCode: BAD_REQUEST - error: - code: FILES_URL_GENERATION_LIMIT_EXCEED - message: 'Pre-signed URL generation failed: limit exceeded.' - trace: '' - /v2/obsrv/data/sql-query: - post: - tags: - - Dataset CRUD APIs - summary: SQL Wrapper - description: This api is a wrapper api, used to query the data. - requestBody: - content: - application/json: - schema: - type: object - example: - query: >- - SELECT "channel","flags", COUNT(*) AS "Count" FROM "wikipedia" - WHERE "page" IS NOT NULL GROUP BY 1, 2 ORDER BY 3 DESC - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - example: - - channel: '#en.wikipedia' - flags: '' - Count: 4776 - - channel: '#fr.wikipedia' - flags: '!M' - Count: 31 - - channel: '#fa.wikipedia' - flags: B - Count: 30 - - channel: '#vi.wikipedia' - flags: 'N' - Count: 9 - - channel: '#ar.wikipedia' - flags: '!N' - Count: 8 - - channel: '#da.wikipedia' - flags: '!' - Count: 8 - - channel: '#hy.wikipedia' - flags: '!N' - Count: 8 - - channel: '#id.wikipedia' - flags: '!N' - Count: 8 - - channel: '#eu.wikipedia' - flags: M - Count: 3 - - channel: '#fa.wikipedia' - flags: '!N' - Count: 3 - - channel: '#fi.wikipedia' - flags: 'N' - Count: 3 - - channel: '#he.wikipedia' - flags: NB - Count: 3 - - channel: '#hi.wikipedia' - flags: '' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!N' - Count: 1 + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c2467e01-0a2d-401c-aa3d-dd16b804f723 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 '400': description: Bad Request - content: - application/json: + headers: + X-Powered-By: schema: - type: object - example: - id: api.obsrv.data.sql-query - ver: v2 - ts: '2024-05-30T16:31:57+05:30' - params: - status: FAILED - resmsgid: 7709ef5b-d7f3-48a5-bf6c-cf243fb6d2eb - responseCode: BAD_REQUEST - error: - code: SQL_QUERY_EMPTY_REQUEST - message: Failed to query. Invalid request - trace: '' - '500': - description: Internal Server Error + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '399' + ETag: + schema: + type: string + example: W/"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:43:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object example: - id: api.obsrv.data.sql-query + id: api.connectors.list ver: v2 - ts: '2024-05-29T13:03:44+05:30' + ts: '2024-07-30T15:13:14+05:30' params: status: FAILED - resmsgid: 0b4b7168-2b70-406b-84ce-c320b65559eb - responseCode: INTERNAL_SERVER_ERROR + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fadde0-8c59-4420-8ab3-56474b01670b + responseCode: BAD_REQUEST + result: {} error: - code: ERR_BAD_RESPONSE - message: Request failed with status code 500 - trace: '' - /v2/datasets/status-transition: - post: + code: CONNECTORS_LIST_INPUT_INVALID + message: >- + #properties/request/properties/filters/properties/status/minItems + must NOT have fewer than 1 items + /v2/connectors/read/{connector_id}: + parameters: + - name: connector_id + in: path + required: true + schema: + type: string + description: Unique identifier for the connector + get: tags: - - Dataset CRUD APIs - summary: Dataset Status Transition - description: >- - This API allows you to perform status transition between 2 states. - Allowed status transition are: -
- - Draft to ReadyToPublish when the dataset configuration is complete and configuration is validated, ReadyToPublish to Live to make dataset live and make it actively running within the pipeline, Live to Retired when dataset is needed to be stopped and retired.User can even Delete the dataset of Draft and ReadyToPublish status. - requestBody: - content: - application/json: - examples: - example1: - summary: Transition to ReadyToPublish - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: ReadyToPublish - example2: - summary: Transition to Live - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Live - example3: - summary: Transition to Retired - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Retire - example4: - summary: Delete a draft dataset - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Delete + - Connector api's + summary: Connector Read responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '3304' + ETag: + schema: + type: string + example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 12:47:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete success: Deleted dataset successfully' + summary: 'Success: Read live Connector' value: - id: api.datasets.status + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:18:54+05:30' + ts: '2024-07-31T18:17:54+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced responseCode: OK result: - message: Dataset deleted successfully - dataset_id: master.1 + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: + type: object + properties: + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: + type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly + fieldDescription: + - type: string + description: '' + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: + batch_size: + ui:readonly: true + max_batches: + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' example-1: - summary: 'Live success: Dataset published successfully' + summary: 'Success: Read Draft Connector' value: - id: api.datasets.status + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:21:42+05:30' + ts: '2024-08-01T12:47:12+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + resmsgid: b6fcfb05-246c-4a1b-9eb1-27497ee9b80b responseCode: OK result: - message: Dataset published successfully - dataset_id: telemetry - example-2: - summary: 'ReadyToPublish success: Dataset is ready to publish' + id: mssql-connector-2.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 2.0.0 + description: >- + The MS SQL Connector is used to move data from any MS + SQL Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Draft + ui_spec: {} + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '276' + ETag: + schema: + type: string + example: W/"114-izVC8DsHdeSfau/USVJvnIqZIMQ" + Date: + schema: + type: string + example: Thu, 01 Aug 2024 09:32:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T15:02:48+05:30' + params: + status: FAILED + resmsgid: 712e7298-99f8-4694-9011-4232fcfd664a + responseCode: NOT_FOUND + result: {} + error: + code: CONNECTOR_NOT_FOUND + message: 'Connector not found: postgres-conn' + /v2/data/in/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset + post: + tags: + - Data Ingest + summary: Data ingest + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + data: + eid: INTERACT + date: '2022-01-01' + ver: '3.0' + syncts: 1668591949682 + ets: 1668591949682 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '261' + ETag: + schema: + type: string + example: W/"105-UtaCmh0qZMBRBdniNq74Gr+4YAo" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:05:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Data ingested successfully' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:30:04+05:30' + ts: '2024-05-30T23:35:48+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 9a44ac5b-ef82-46f7-92c5-c5b39ef88764 responseCode: OK result: - message: Dataset status transition to ReadyToPublish successful - dataset_id: telemetry-events - example-3: - summary: 'Retire success: Dataset retired successfully' + message: Data ingested successfully + example-1: + summary: 'Success: Data ingested successfully (batch)' value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T12:22:58+05:30' + ts: '2024-05-30T23:36:49+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: a220041c-0c28-415a-9687-9fb2e211f8c4 responseCode: OK result: - message: Dataset retired successfully - dataset_id: sb-telemetry + message: Data ingested successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '300' + ETag: + schema: + type: string + example: W/"12c-GNCIUUDxOZD3UfM311sjnmFIgPc" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:03:32 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '2024-05-30T23:33:32+05:30' + params: + status: FAILED + resmsgid: acf07609-77de-4ab5-81ea-42e41b8b95ff + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_INGESTION_INVALID_INPUT + message: '#required should have required property ''id''' + trace: '' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '316' + ETag: + schema: + type: string + example: W/"13c-O9iirC/EyneYXQzth7iwEEy1UV4" + Date: + schema: + type: string + example: Thu, 18 Apr 2024 10:17:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Live failure: Dataset in draft state' + summary: Entry topic not found value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T15:37:43+05:30' + ts: '2024-04-18T15:47:00+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 11c32a3a-fdeb-4e00-a9cf-f6433ade5608 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_NOT_READY_FOR_PUBLISH - message: Failed to publish dataset as it is in draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' example-1: - summary: 'ReadyToPublish failure: Incomplete dataset configs' + summary: 'Failure: Dataset not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:36:16+05:30' + ts: '2024-05-30T23:37:29+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: d4db36b4-37b0-4170-a4cf-9d2ae8fa0416 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_CONFIGS_INVALID - message: >- - #properties/denorm_config/properties/denorm_fields/items/required - must have required property 'dataset_name' + code: DATASET_NOT_FOUND + message: Dataset with id not found trace: '' example-2: - summary: 'ReadyToPublish failure: Dataset not in draft state' + summary: 'Failure: Entry topic not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:38:14+05:30' + ts: '2024-05-30T23:39:26+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 62309380-3e83-442f-877e-359ed2774bbf responseCode: BAD_REQUEST + result: {} error: - code: DATASET_READYTOPUBLISH_FAILURE - message: >- - Failed to mark dataset Ready to publish as it not in - draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' - example-3: - summary: 'Retire Failure: Dataset is already retired' + /v2/data/query/test: + post: + tags: + - Data query + summary: Data query + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + context: + aggregationLevel: day + query: >- + Select * from "check" WHERE __time >= timestamp '2020-12-31' + AND __time < TIMESTAMP '2024-01-21' Limit 1 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '313' + ETag: + schema: + type: string + example: W/"139-De+IthAwrGNR+J11CwlNf5RSMmw" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:17:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: native query (interval as array type)' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:42:18+05:30' + ts: '2024-05-30T23:47:45+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_RETIRED - message: Dataset is already retired - trace: '' - example-4: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 134efe35-c096-4cab-ad14-db6a8952f264 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-1: + summary: 'Success: native query (queryType: scan)' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:51:14+05:30' + params: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 4fc6e3c1-4715-47b7-9137-7713fb2fbe72 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-2: summary: >- - Retire failure: Cannot retire master dataset as it is used - by other datasets + Success: native query (normal intervals & queryType : + timeseries) value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:01:41+05:30' + ts: '2024-05-30T23:51:49+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 - responseCode: BAD_REQUEST - error: - code: DATASET_IN_USE - message: Failed to retire dataset as it is used by other datasets - trace: '' - example-5: - summary: 'Failure: Invalid request payload provided' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75d7ae48-a3db-4367-b50f-34eb99ac3480 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + school_id: 0 + example-3: + summary: 'Success: sql query' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:03:56+05:30' + ts: '2024-05-30T23:42:12+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 - responseCode: BAD_REQUEST - error: - code: DATASET_STATUS_INVALID_INPUT - message: >- - #properties/request/properties/status/enum should be - equal to one of the allowed values - trace: '' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 2c981011-76da-3000-97f3-eafac939e59f + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '371' + ETag: + schema: + type: string + example: W/"173-OP8NcbSqLKFO92PIyUmMk0lNsXs" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:16:29 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:46:29+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: b6434700-dd92-4f64-9250-a22939e753b9 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_OUT_INVALID_DATE_RANGE + message: >- + Invalid date range! make sure your range cannot be more than + 30 days + trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '356' + ETag: + schema: + type: string + example: W/"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:13:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete failure: Dataset not found to delete' + summary: 'Failure: Datasource not found in live table' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T12:25:36+05:30' + ts: '2024-05-30T23:43:12+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: f922f120-2aea-49af-9a76-7312fe2eb266 responseCode: NOT_FOUND + result: {} error: - code: DATASET_NOT_FOUND - message: Dataset not found to delete + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Datasource telemetry-eventssss not available for + querying trace: '' example-1: - summary: 'Live failure: Dataset not found to publish' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T15:35:59+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c - responseCode: NOT_FOUND - error: - code: DATASET_NOT_FOUND - message: Dataset not found to publish - trace: '' - example-2: - summary: 'Retire Failure: Dataset not found to retire' + summary: 'Failure: Datasource not found in druid' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:40:31+05:30' + ts: '2024-05-30T23:50:27+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75bd4313-d504-4fd3-92ab-ee2a685beb83 responseCode: NOT_FOUND - error: - code: DATASET_NOT_FOUND - message: Dataset not found to retire - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T16:24:12+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 92928434-719f-47d4-9946-1e40ecd53253 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_STATUS_FAILURE - message: Failed to perform status transition on datasets - trace: '' - /v2/data/in/{datasetId}: - post: - tags: - - Data Ingest - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - summary: Data ingest - description: >- - This API allows you to ingest events to a data streaming platform in individual or batch format. - requestBody: - content: - application/json: - examples: - example1: - summary: Ingest individual event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: - eid: INTERACT - date: '2022-01-01' - ver: '3.0' - syncts: 1668591949682 - ets: 1668591949682 - example2: - summary: Ingest batch event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: [{ "eid": "INTERACT","date": "2022-01-01","ver": "3.0","syncts": 1668591949682}] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Data ingest successful(individual) - value: - id: api.data.in - ver: v2 - ts: 1711966306164 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - example-1: - summary: Data ingest successfully(batch) - value: - id: api.data.in - ver: v2 - ts: 1711966352987 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.data.in - ver: v2 - ts: 1711966410766 - params: - status: FAILED - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATA_INGESTION_INVALID_INPUT - message: "#required should have required property 'id'" - trace: '' - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Dataset not found - value: - id: api.data.in - ver: v2 - ts: 1711966446364 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_FOUND - message: Dataset with id not found - trace: '' - example-1: - summary: Entry topic not found - value: - id: api.data.in - ver: v2 - ts: 1711975482739 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: ENTRY_TOPIC_NOT_FOUND - message: Entry topic is not defined - trace: '' - /v2/data/query/{datasetId}: - post: - tags: - - Data query - summary: Data query - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - description: >- - This API allows you to query your datasource. - requestBody: - content: - application/json: - examples: - example1: - summary: SQL query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: >- - SELECT * FROM "rollups-configured" WHERE __time >= TIMESTAMP - '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1 - example2: - summary: Native query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - mid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: - quertType: scan - dataSource: rollups-configured - intervals: - type: intervals - intervals: ["2022-01-01/2022-02-01"] - granularity: day - columns: ["eid","__time"] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: sql query' - value: - id: api.data.out - ver: v2 - ts: 1711971972855 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - - __time: '2022-01-01T00:00:00.000Z' - eid: '1' - ver: '3.0' - flags_ex_processed: 'true' - flags_pp_validation_processed: 'true' - flags_pp_duplicate_skipped: 'true' - flags_device_denorm: 'true' - flags_user_denorm: 'true' - flags_loc_denorm: 'true' - derivedlocationdata_district: AGRA - derivedlocationdata_from: user-profile - derivedlocationdata_state: Uttar Pradesh - mid: 6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0 - type: events - actor_type: User - actor_id: 311663b2-d7de-4d46-8803-20407eaa3403 - edata_type: session - userdata_subject: null - userdata_district: AGRA - userdata_usersubtype: hm - userdata_grade: null - userdata_usersignintype: Self-Signed-In - userdata_usertype: administrator - userdata_userlogintype: administrator - userdata_state: Uttar Pradesh - devicedata_statecustomcode: '29' - devicedata_country: India - devicedata_iso3166statecode: IN-KA - devicedata_city: Bengaluru - devicedata_countrycode: IN - devicedata_state: Karnataka - devicedata_devicespec_idisk: '106.47' - devicedata_devicespec_webview: 107.0.5304.105 - devicedata_devicespec_os: Android 12 - devicedata_devicespec_scrn: '6.53' - devicedata_devicespec_sims: '-1' - devicedata_devicespec_cpu: "abi: arm64-v8a processor\t: 0 " - devicedata_devicespec_id: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - devicedata_devicespec_camera: '' - devicedata_devicespec_edisk: '106.27' - devicedata_devicespec_make: vivo 1915 - devicedata_statecode: KA - devicedata_districtcustom: BENGALURU URBAN SOUTH - devicedata_statecustomname: Karnataka - devicedata_userdeclared_district: AGRA - devicedata_userdeclared_state: Uttar Pradesh - context_cdata_id: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_cdata_type: UserSession - context_env: sdk - context_channel: '0126796199493140480' - context_pdata_id: preprod.diksha.app - context_pdata_pid: sunbird.app - context_pdata_ver: 4.10.1023preproduction - context_sid: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_did: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - context_rollup_l1: '0126796199493140480' - object_id: '' - object_type: '' - object_version: '' - object_rollup: '{}' - count: 3 - edata_duration_sum: null - example-1: - summary: 'Success: native query' - value: - id: api.data.out - ver: v2 - ts: 1711973520793 - params: - status: SUCCESS - errmsg: '' - responseCode: OK - result: - - timestamp: '2022-01-01T00:00:00.000Z' - result: - mid: 1 - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: invalid date range (sql)' - value: - id: api.data.out - ver: v2 - ts: 1711972193426 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - example-1: - summary: 'Failure: Invalid date range (native)' - value: - id: api.data.out - ver: v2 - ts: 1711972501847 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.data.out - ver: v2 - ts: 1711972085550 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.data.out - ver: v2 - ts: 1711975034302 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-2: - summary: 'Failure: Datasource not found in live table (native)' - value: - id: api.data.out - ver: v2 - ts: 1711974978635 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" - example-3: - summary: 'Failure: Datasource not found in druid(native)' - value: - id: api.data.out - ver: v2 - ts: 1711972569115 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-2: + summary: 'SQL Failure: dataset not found in druid' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:52:26+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 19849769-c290-4599-a79e-862dcbce1124 + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-3: + summary: 'SQL Failure: Datasource not found in live table' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:53:02+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 530a12f9-04f3-4986-8f52-b3184b9194bd + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource undefined not available for querying + trace: '' /v2/template/read/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template get: tags: - Query Templates summary: Read template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '537' + ETag: + schema: + type: string + example: W/"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:24:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3362,10 +7986,10 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:09:55+05:30' + ts: '2024-05-30T23:54:14+05:30' params: status: SUCCESS - resmsgid: 8ef3a9d0-8658-47dd-b6cb-a4b1777a1ea4 + resmsgid: ad1c0a98-1ba3-4203-9fac-59ccaf442347 responseCode: OK result: template_id: jsontemplate111 @@ -3382,24 +8006,54 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:10:54+05:30' + ts: '2024-05-30T23:54:38+05:30' params: status: SUCCESS - resmsgid: 656185ef-527d-4b60-ac19-263da85142b8 + resmsgid: 70047866-a4b4-4fce-b9c4-4699fcab2dc6 responseCode: OK result: template_id: sql1 template_name: sql1 query: >- - "SELECT * FROM {{DATASET}} WHERE __time BETWEEN - TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP + {{ENDDATE}} lIMIT {{LIMIT}}" query_type: sql created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-04-29T11:29:58.759Z' - updated_date: '2024-04-29T11:29:58.759Z' + created_date: '2024-05-13T07:29:04.117Z' + updated_date: '2024-05-13T07:29:04.117Z' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '294' + ETag: + schema: + type: string + example: W/"126-zq5j1yor+xr1XFEEbP09fQTDI/k" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:25:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3407,30 +8061,60 @@ paths: example: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:11:45+05:30' + ts: '2024-05-30T23:55:16+05:30' params: status: FAILED - resmsgid: bc6a4621-4ac5-4959-b467-1134ea874960 + resmsgid: 958bf0e7-bdb8-4153-abdc-ab84e8004a0e responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template sql100 does not exists trace: '' /v2/template/delete/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template delete: tags: - Query Templates summary: Delete template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '241' + ETag: + schema: + type: string + example: W/"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo" + Date: + schema: + type: string + example: Tue, 30 Apr 2024 06:28:27 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3447,6 +8131,35 @@ paths: message: Template yashash-k deleted successfully '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '303' + ETag: + schema: + type: string + example: W/"12f-99pWw8VTwuVfDAhinC55JXfNyyg" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:28:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3454,27 +8167,21 @@ paths: example: id: api.query.template.delete ver: v2 - ts: '2024-04-29T18:29:09+05:30' + ts: '2024-05-30T23:58:41+05:30' params: status: FAILED - resmsgid: 6c62ff4d-0a63-4384-a365-7bbef951f288 + resmsgid: 9b7f81fb-6705-4d32-9bd3-139cd5a211b9 responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template json_template does not exists trace: '' - /v2/template/create/{template_id}: + /v2/template/create: post: tags: - Query Templates summary: Create query template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path requestBody: content: application/json: @@ -3493,9 +8200,44 @@ paths: datasetId: '{{DATASET}}' intervals: '{{STARTDATE}}/{{ENDDATE}}' limit: '{{LIMITS}}' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '359' + ETag: + schema: + type: string + example: W/"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 16:59:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3533,6 +8275,35 @@ paths: message: The query template has been saved successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '407' + ETag: + schema: + type: string + example: W/"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:05:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3549,6 +8320,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 62e18342-7e25-4122-8fca-6fb12fac3ff0 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID message: >- @@ -3566,6 +8338,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID_INPUT message: >- @@ -3574,6 +8347,35 @@ paths: trace: '' '409': description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '350' + ETag: + schema: + type: string + example: W/"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:03:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3587,6 +8389,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 18b6b123-4df5-4124-b6ec-73b667250e1c responseCode: CONFLICT + result: {} error: code: QUERY_TEMPLATE_ALREADY_EXISTS message: Template josnaks-aaa already exists @@ -3616,6 +8419,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '6864' + ETag: + schema: + type: string + example: W/"1ad0-xp24UiXXXiFWplmv5Acja7prSYM" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 13:46:03 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4176,17 +9008,17 @@ paths: created_date: '2024-04-25T06:31:20.819Z' updated_date: '2024-04-25T06:31:20.819Z' /v2/template/update/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template patch: tags: - Query Templates summary: update template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path requestBody: content: application/json: @@ -4207,6 +9039,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '314' + ETag: + schema: + type: string + example: W/"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:51:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4214,341 +9075,929 @@ paths: example: id: api.query.template.update ver: v2 - ts: '2024-05-02T13:16:31+05:30' + ts: '2024-05-10T11:21:47+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ab869754-d2f0-4b9a-b38b-04d9e5901b5d + resmsgid: 9e4a6959-0eb9-4fc4-8e6f-2eea534d1384 responseCode: OK result: message: Query template updated successfully templateId: sql11template1 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '429' + ETag: + schema: + type: string + example: W/"1ad-5sb8WUekFL8s4c1Ink6bUByoHho" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:53:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: required variables not exists to update' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:23:54+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 66b95cb3-2ef2-4735-9045-2674da552dbd + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Invalid template provided, A template should consist of + variables DATASET,STARTDATE,ENDDATE and type of json,sql + trace: '' + example-1: + summary: 'Failure: Template name validation failure' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:26:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: d6428fcf-53c9-465d-9431-769218f775b8 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Template name should contain alphanumeric characters and + single space between characters + trace: '' + example-2: + summary: 'Failure: query_type should when updating query' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:32:57+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c7a8675a-73f2-4764-abba-bfdf9f8b4621 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query_type when property query is present + trace: '' + example-3: + summary: 'Failure: query should present when updating query_type' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:34:55+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 2c1098b2-d7b3-4d39-98ee-e3e790fd23b4 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query when property query_type is present + trace: '' + /v2/template/query/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template + post: + tags: + - Query Templates + summary: query template + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.query + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + startdate: '2020-12-31' + enddate: '2024-12-31' + aggregationLevel: month + dataset: test + limit: 5 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-9hKB38iHEwYPT2MgF8puXcq05Ew" + Date: + schema: + type: string + example: Tue, 14 May 2024 06:22:24 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: JSON template with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:52:24+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + responseCode: OK + result: + - timestamp: '2023-09-01T00:00:00.000Z' + result: + school_id: 0 + example-1: + summary: 'Success: SQL template query with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:33:06+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '336' + ETag: + schema: + type: string + example: W/"150-T/XeSIt7PR7GcGEbET1e8n9zX7k" + Date: + schema: + type: string + example: Thu, 02 May 2024 07:29:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: query should present when query_type provided' + summary: 'Failure: invalid date range (native template)' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:30:36+05:30' + ts: '2024-05-02T12:59:14+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: f11f5287-77cd-43f0-b7f6-6d6b94b9cea2 + resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 responseCode: BAD_REQUEST + result: {} error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - #properties/request/dependencies should have property - query when property query_type is present + Invalid date range! make sure your range cannot be more + than 30 days trace: '' example-1: - summary: 'Failure: query_type should present when query is given' - value: - id: api.query.template.update - ver: v2 - ts: '2024-05-02T11:33:23+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 937cf790-867b-450b-afc9-86a96d02f636 - responseCode: BAD_REQUEST - error: - code: QUERY_TEMPLATE_INVALID_INPUT - message: >- - #properties/request/dependencies should have property - query_type when property query is present - trace: '' - example-2: - summary: 'Failure: template name validation failure' + summary: 'Failure: invalid date range' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:34:18+05:30' + ts: '2024-05-13T13:28:18+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9f3ae5aa-2e98-4204-8f87-b11e5bb3865a + resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 responseCode: BAD_REQUEST error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - Template name should contain alphanumeric characters and - single space between characters + Invalid date range! make sure your range cannot be more + than 30 days trace: '' - example-3: - summary: 'Failure: required template_id' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '357' + ETag: + schema: + type: string + example: W/"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U" + Date: + schema: + type: string + example: Mon, 13 May 2024 07:51:46 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Datasource not found in druid' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:38:16+05:30' + ts: '2024-05-13T13:21:46+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 94b71f80-bd1e-4283-8f3d-83833b60d08c - responseCode: BAD_REQUEST + resmsgid: b35a7050-b94c-4944-9630-233c9542272e + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_SOURCE_NOT_FOUND message: >- - #properties/request/required should have required - property 'template_id' + Dataset test with table hour is not available for + querying trace: '' - example-4: - summary: 'Failure: required variables not exists' + example-1: + summary: 'Failure: Datasource not found in live table' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T13:13:22+05:30' + ts: '2024-05-13T13:23:47+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dff3c22a-c198-4d1c-b87a-07d725f868f7 - responseCode: BAD_REQUEST + resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID - message: >- - Invalid template provided, A template should consist of - variables DATASET,STARTDATE,ENDDATE and type of json,sql + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource test11 not available for querying trace: '' - /v2/template/query/{template_id}: + /alerts/v1/notifications/search: + post: + tags: + - Alert Notification-channels + summary: search channels + requestBody: + content: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/create: + post: + tags: + - Alert Notification-channels + summary: Add channel (Email) + requestBody: + content: + application/json: + schema: + type: object + example: + manager: grafana + name: functional-metrics-email + type: email + config: + recipientAddresses: yravinderkumar33@gmail.com;ravinder@sanketika.in + subject: Obsrv Prod Alert + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert Notification-channels + summary: publish channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/test: + post: + tags: + - Alert Notification-channels + summary: test channel + requestBody: + content: + application/json: + schema: + type: object + example: + message: Testing Email integration. If you can read this, it's working! + payload: + error: + selectChannel: true + configureChannel: true + manager: grafana + name: udhw + type: email + config: + recipientAddresses: jerald@sanketika.in + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Notification-channels + summary: update channel + requestBody: + content: + application/json: + schema: + type: object + example: + name: updated name + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Notification-channels + summary: delete channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/create: + post: + tags: + - Alert silence + summary: Add Silence + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2024-01-11T07:56:23Z' + endDate: '2024-01-11T08:56:23Z' + alertId: c7464d32-1d8d-4eaf-9b23-1313a3ff8149 + manager: grafana + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert silence + summary: Get Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert silence + summary: Delete Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/search: + get: + tags: + - Alert silence + summary: Fetch All Silences + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert silence + summary: Edit Request + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2023-08-09T10:50:59Z' + endDate: '2023-08-10T10:30:59Z' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alerts Wrapper + summary: 'delete rule ' + description: This URLwill provided access to user to delete any custom rule. + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alerts Wrapper + summary: 'edit rule ' + description: >- + This URL will provide access to users to edit any properties in the + Alert rule and save the changes. + requestBody: + content: + application/json: + schema: + type: object + example: + description: >- + This alert rule is designed to promptly notify you when one or + more servers in your infrastructure become unresponsive or + inaccessible. + labels: + severity: warning + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'publish rule ' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/create: + post: + tags: + - Alerts Wrapper + summary: 'Add Rule ' + requestBody: + content: + '*/*': + schema: + type: string + example: >- + "{\n \"name\": \"Total Api Calls1\",\n + \"manager\":\"grafana\",\n \"description\": \"This alert is + set up to notify you when the CPU usage on a host reaches a Low + level.\",\n \"expression\": \"(node_total_api_calls) > + 20\",\n \"category\": \"Infra\",\n + \"severity\":\"warning\",\n \"frequency\": \"1m\",\n + \"interval\": \"1m\",\n \"labels\": {\n \"component\": + \"api\",\n \"notificationChannel\": \"slack\"\n + },\n \"annotations\":{\n \"summary\":\"Host Low CPU + usage\"\n },\n \"metadata\": {\n \"query\": + [\n {\n \"refId\": + \"A\",\n \"datasourceUid\": + \"$datasourceUid\",\n \"queryType\": + \"\",\n \"relativeTimeRange\": + {\n \"from\": 600,\n + \"to\": 0\n },\n \"model\": + {\n \"refId\": \"A\",\n + \"hide\": false,\n \"editorMode\": + \"code\",\n \"expr\": + \"node_total_api_calls\",\n \"legendFormat\": + \"__auto\",\n \"range\": + true\n }\n },\n + {\n \"refId\": \"B\",\n + \"datasourceUid\": \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"B\",\n \"hide\": + false,\n \"type\": + \"reduce\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"B\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"reducer\": + \"last\",\n \"expression\": + \"A\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n },\n {\n \"refId\": + \"C\",\n \"datasourceUid\": + \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"C\",\n \"hide\": + false,\n \"type\": + \"threshold\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [\n + 20\n + ],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"C\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"expression\": + \"B\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n }\n ]\n }\n}" + parameters: + - name: Accept + in: header + schema: + type: string + example: application/json, text/plain, */* + - name: Accept-Language + in: header + schema: + type: string + example: en-GB,en + - name: Cache-Control + in: header + schema: + type: string + example: no-store + - name: Connection + in: header + schema: + type: string + example: keep-alive + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; + grafana_session=4eb514f6fef4ad6884e47e50254af650; + grafana_session_expiry=1687239735 + - name: Origin + in: header + schema: + type: string + example: http://localhost:3001 + - name: Pragma + in: header + schema: + type: string + example: no-store + - name: Referer + in: header + schema: + type: string + example: http://localhost:3001/alertRules/add + - name: Sec-Fetch-Dest + in: header + schema: + type: string + example: empty + - name: Sec-Fetch-Mode + in: header + schema: + type: string + example: cors + - name: Sec-Fetch-Site + in: header + schema: + type: string + example: same-origin + - name: Sec-GPC + in: header + schema: + type: integer + example: '1' + - name: User-Agent + in: header + schema: + type: string + example: >- + Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like + Gecko) Chrome/114.0.0.0 Safari/537.36 + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/search: post: tags: - - Query Templates - summary: query template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + - Alerts Wrapper + summary: 'search rule ' requestBody: content: application/json: schema: type: object example: - id: api.query.template.query - ver: v2 - ts: '2024-04-10T16:10:50+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - request: - startdate: '2020-12-31' - enddate: '2024-12-31' - aggregationLevel: month - dataset: test - limit: 5 + request: {} responses: '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: JSON template with request body' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-14T11:52:24+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d - responseCode: OK - result: - - timestamp: '2023-09-01T00:00:00.000Z' - result: - school_id: 0 - example-1: - summary: 'Success: SQL template query with request body' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-14T11:33:06+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 - responseCode: OK - result: - - __time: '2023-09-11T00:00:00.000Z' - school_category: secondary - gender: others - state_id: '15' - district_id: '2002' - block_id: '70' - cluster_id: '485' - obsrv.meta.source.connector: null - obsrv.meta.source.id: null - grade_sum: 18 - school_id_sum: 180378 - students_marked_sum: 12492 - students_present_sum: 2466 - total_count: 18 - total_students_sum: 12492 - '400': - description: Bad Request + description: Successful response content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: invalid date range (native template)' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-02T12:59:14+05:30' - params: - status: FAILED - resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' - example-1: - summary: 'Failure: invalid date range' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:28:18+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' - '404': - description: Not Found + application/json: {} + /alerts/v1/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'get specific rule ' + responses: + '200': + description: Successful response content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:21:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b35a7050-b94c-4944-9630-233c9542272e - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: >- - Dataset test with table hour is not available for - querying - trace: '' - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:23:47+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: Datasource test11 not available for querying - trace: '' - /v2/dataset/copy: + application/json: {} + /alerts/v1/metric/alias/create: post: tags: - - Dataset copy - summary: Dataset copy + - Alert Metric_alias + summary: add metric requestBody: content: application/json: schema: type: object example: - id: api.datasets.copy - ver: '1.0' - ts: '2024-05-21T14:30:00Z' - params: - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - request: - source: - datasetId: test_dataset - isLive: true - destination: - datasetId: test-dataset-copy-1 + alias: Druid + component: MyTest + metric: sum(druid_supervisors{state='SUSPENDED'}) responses: '200': - description: OK + description: Successful response content: - application/json: - schema: - type: object - example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:57:11+05:30' - params: - status: SUCCESS - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: 77080c18-67b3-489c-af7b-9739709e3c4b - responseCode: OK - result: - dataset_id: sample1_copy.1 - message: Dataset clone successful - '400': - description: Bad Request + application/json: {} + /alerts/v1/metric/alias/search: + post: + tags: + - Alert Metric_alias + summary: list metrics + requestBody: + content: + application/json: + schema: + type: object + example: {} + responses: + '200': + description: Successful response content: - application/json: - schema: - type: object + application/json: {} + /alerts/v1/metric/alias/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Metric_alias + summary: update metric + requestBody: + content: + application/json: + schema: + type: object example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:56:26+05:30' - params: - status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_EXISTS - message: Dataset with id sample1 already exists - trace: '' - '404': - description: Not Found + alias: Druid native + responses: + '200': + description: Successful response content: - application/json: - schema: - type: object - example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:59:59+05:30' - params: - status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: NOT_FOUND - error: - code: DATASET_NOT_EXISTS - message: Dataset sample does not exists - trace: '' + application/json: {} + /alerts/v1/metric/alias/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Metric_alias + summary: remove metric + responses: + '200': + description: Successful response + content: + application/json: {} \ No newline at end of file From aae31f95f27142b5f7279fade1239664883e25a8 Mon Sep 17 00:00:00 2001 From: yashash <126703764+yashashkumar@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:04:19 +0530 Subject: [PATCH 220/311] #OBS-I352 : updated swagger doc (#280) --- api-service/swagger-doc/openapi_v2.yml | 11395 +++++++++++++++++------ 1 file changed, 8422 insertions(+), 2973 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index d4cd25b5..21dc9e3e 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -1,36 +1,38 @@ openapi: 3.0.0 info: - title: Obsrv API V2 + title: Obsrv v2 apis description: >- Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets. version: 1.0.0 servers: - - url: ":3000" + - url: http://localhost:3000 + - url: http://{{host_ip}} tags: - - name: Dataset CRUD APIs + - name: Dataset api's description: >- The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria. + - name: Connector api's - name: Data Ingest - name: Data query - - name: Data exhaust - name: Query Templates - - name: Dataset copy - - name: Schema validator + - name: Alert Notification-channels + - name: Alert silence + - name: Alerts Wrapper + - name: Alert Metric_alias paths: /v2/datasets/create: post: tags: - - Dataset CRUD APIs + - Dataset api's summary: Dataset create description: >- This API allows you to create new datasets used by the analytical data source. requestBody: - required: true content: application/json: schema: @@ -42,8 +44,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - type: dataset + dataset_id: telemetry_record-t4 + type: event name: sb-telemetry validation_config: validate: true @@ -78,148 +80,418 @@ paths: denorm_fields: - denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry transformations_config: - - field_key: eid + - field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - dataset_config: - data_key: mid - timestamp_key: ets - files_upload_path: ["telemetry.json"] tags: - tag1 - - tag2 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '287' + ETag: + schema: + type: string + example: W/"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ" + Date: + schema: + type: string + example: Mon, 15 Jul 2024 13:14:09 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset Success + summary: 'Success: Dataset created successfullly' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-15T18:44:08+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 276c042c-0f23-4b26-9b10-6fe48bbc2d3a responseCode: OK result: - id: sb-telemetry.1 - version_key: '1713442037275' + id: telemetry_record-t4 + version_key: '1721049248930' example-1: - summary: Master Dataset Success + summary: 'Success: Master dataset created successfully' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:36:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 845076be-d9e7-4246-bb8e-07ae0ce59d1e + responseCode: OK + result: + id: telemetry_record-master + version_key: '1721099200603' + example-2: + summary: 'Success: Minimal request body' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T18:14:59+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9e207f4f-2be6-4a45-ab78-213bea272ae0 + responseCode: OK + result: + id: telemetry_events + version_key: '1721133899306' + example-3: + summary: 'Success: Dataset created successfully with all fields' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-17T18:19:53+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 505fb3bc-ae32-4f5b-a931-adec4d1d84ba responseCode: OK result: - id: sb-telemetry-master.1 - version_key: '1713442037275' + id: telemetry_record-t41 + version_key: '1721220593027' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '362' + ETag: + schema: + type: string + example: W/"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:09:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:39:00+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a07de860-dcbc-4ff6-822e-34b47635c8a3 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '337' + ETag: + schema: + type: string + example: W/"151-a7dJ9XBUyT3AXNxl1TPcraxMX08" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:07:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Dataset contains duplicate denorm out field' + summary: 'Failure: Master dataset already exists' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:59:06+05:30' + ts: '2024-07-16T08:37:28+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 54c5b692-dc37-432e-b556-7f396d7c9e07 - responseCode: BAD_REQUEST + resmsgid: 138b796b-1b68-481a-a59d-1cb695c1adc9 + responseCode: CONFLICT + result: {} error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Duplicate denorm key found - trace: '' + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-master example-1: - summary: 'Failure: Invalid request payload provided' + summary: 'Failure: Dataset already exists' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T18:00:34+05:30' + ts: '2024-07-16T08:38:05+05:30' params: status: FAILED - resmsgid: 615c1e4c-8c19-44fd-b29c-c235e7cbb5f0 - responseCode: BAD_REQUEST + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf62693c-3aa4-42ce-a5ea-4bde340740f5 + responseCode: CONFLICT + result: {} error: - code: DATASET_INVALID_INPUT - message: >- - #additionalProperties should NOT have additional - properties - trace: '' - '409': - description: Conflict + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-t4 + /v2/files/generate-url: + post: + tags: + - Dataset api's + summary: File generate url + description: This API generates presigned URLs to upload or download files from cloud + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.files.generate-url + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '1344' + ETag: + schema: + type: string + example: W/"540-790rZel+H/rDwgvZRxvlUmZ8Gpc" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 02:56:19 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T17:56:06+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 76612ad1-241b-4ce3-8af4-88db860697f4 - responseCode: CONFLICT - error: - code: DATASET_EXISTS - message: Dataset already exists - trace: '' - '500': - description: Internal Server Error + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '355' + ETag: + schema: + type: string + example: W/"163-9oQYJJEaBH3mJAnzDHXn2MxE848" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:03:04 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T18:02:44+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dd1c0e11-fb4c-484c-81fe-82c9e2eee053 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_CREATION_FAILURE - message: Failed to create dataset - trace: '' + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' /v2/datasets/update: patch: tags: - - Dataset CRUD APIs + - Dataset api's summary: Dataset update description: >- This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or - update transformations + update transformations and connectors requestBody: - required: true content: application/json: schema: @@ -231,8 +503,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - version_key: '1717073955640' + dataset_id: telemetry_record-t4 + version_key: '1721135455988' name: sb-telemetry validation_config: validate: true @@ -242,279 +514,254 @@ paths: extraction_key: events dedup_config: drop_duplicates: true - dedup_key: id + dedup_key: ipid dedup_config: drop_duplicates: true - dedup_key: eid + dedup_key: mid data_schema: $schema: https://json-schema.org/draft/2020-12/schema type: object properties: - eid: + midpid: type: string - ets: + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: type: string - required: - - eid + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string additionalProperties: true denorm_config: denorm_fields: - - values: + - value: denorm_key: eid denorm_out_field: userdata - dataset_id: master-telemetry action: remove - - values: + - value: denorm_key: eid denorm_out_field: edata - dataset_id: trip-record - action: add - transformation_config: - - values: - field_key: eid + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert + - value: + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - action: add - dataset_config: - data_key: eid - timestamp_key: ets - tags: - - values: - - tag1 - - tag2 action: remove - - values: - - tag3 - - tag4 - action: add + tags: [] + connectors_config: + - value: + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice + action: upsert + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '331' + ETag: + schema: + type: string + example: W/"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 13:00:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T00:16:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 4b2c0c22-f765-46f9-80af-33d0442db5ca - responseCode: OK - result: - message: Dataset is updated successfully - id: sb-telemetry.1 - version_key: '1713465973004' + examples: + example-0: + summary: 'Success: Minimal dataset update' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:30:45+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 354f1fec-0c39-42ee-a52a-49552f847c11 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134845559' + example-1: + summary: 'Success: Updated successfully' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:27:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 6d835f07-aed5-4e8b-81c2-2142cfb55c52 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134675878' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '369' + ETag: + schema: + type: string + example: W/"171-iNJoyWUecOEsXbHZwx6rld3Sr1I" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:59:21 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Invalid payload provided' + summary: 'Failure: Invalid request' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:22:12+05:30' + ts: '2024-07-16T18:29:21+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9408e137-ada8-48b8-99e9-90d5cdd35e35 + resmsgid: 7d31672b-e5c3-4a6d-afac-d9d78011bcde responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_INPUT_INVALID message: >- - #properties/request/properties/name/type should be - string - trace: '' + #properties/request/required must have required property + 'dataset_id' example-1: - summary: 'Failure: No field provided along with dataset_id' + summary: 'Failure: No fields are provided to update' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:23:06+05:30' + ts: '2024-07-16T18:32:44+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 347f465f-f100-4095-9801-38c4943380c0 + resmsgid: bf99b1e1-7694-4be0-ba5d-e347764736de responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_NO_FIELDS message: >- Provide atleast one field in addition to the dataset_id to update the dataset - trace: '' - example-2: - summary: 'Failure: Cannot update as dataset not in draft state' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:25:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: a9dfc926-c893-4ec3-82b8-bc8601d928a9 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE - message: Dataset cannot be updated as it is not in draft state - trace: '' - example-3: - summary: 'Failure: Dataset contains duplicate denorm out field' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:29:25+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 865b9bef-9b6d-467c-9065-6689126b0f42 - responseCode: BAD_REQUEST - error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Dataset contains duplicate denorm out keys:[userdata] - trace: '' - example-4: - summary: 'Failure: Dataset tags to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:32:05+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9e9ff0bc-d660-4c4e-a2b0-cb11246a6961 - responseCode: BAD_REQUEST - error: - code: DATASET_TAGS_EXISTS - message: Dataset tags already exist - trace: '' - example-5: - summary: 'Failure: Dataset transformations to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:34:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e0a5cc26-032e-4be8-81ef-c7e76867c9c1 - responseCode: BAD_REQUEST - error: - code: DATASET_TRANSFORMATIONS_EXIST - message: Dataset transformations already exists - trace: '' - example-6: - summary: 'Failure: Dataset denorm fields to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:38:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b807580-a2b5-4a44-adc4-80b35b4b6529 - responseCode: BAD_REQUEST - error: - code: DATASET_DENORM_EXISTS - message: Denorm fields already exist - trace: '' '404': description: Not Found - content: - application/json: + headers: + X-Powered-By: schema: - type: object - examples: - example-0: - summary: 'Failure: Dataset does not exists to update' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:20:39+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ebcf0a62-7c95-40f5-adf2-87ab90a40a80 - responseCode: NOT_FOUND - error: - code: DATASET_NOT_EXISTS - message: Dataset does not exists to update - trace: '' - example-1: - summary: 'Failure: Dataset tags to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:33:33+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: c29e7d0f-3eaf-4972-8804-c172db7b6f13 - responseCode: NOT_FOUND - error: - code: DATASET_TAGS_DO_NOT_EXIST - message: Dataset tags do not exist to remove - trace: '' - example-2: - summary: 'Failure: Dataset transformations to update do not exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:36:04+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b5f515b-1958-4597-b603-b4038d4e5846 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to update - trace: '' - example-3: - summary: 'Failure: Dataset transformations to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:37:20+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 69d32411-4252-4b7f-8afa-6d436928c3d3 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to remove - trace: '' - example-4: - summary: 'Failure: Dataset denorm to remove does not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:40:07+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ce78e393-2b74-46da-a70b-2abd06ad3270 - responseCode: NOT_FOUND - error: - code: DATASET_DENORM_DO_NOT_EXIST - message: Denorm fields do not exist to remove - trace: '' - '409': - description: Conflict + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '340' + ETag: + schema: + type: string + example: W/"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:58:30 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -522,20 +769,47 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-24T12:30:18+05:30' + ts: '2024-07-16T18:28:30+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: fe4a438d-441c-4497-a3ec-a0cee2058b48 - responseCode: CONFLICT + resmsgid: bf64703c-bb6b-41bf-bc1a-c85373efd925 + responseCode: NOT_FOUND + result: {} error: - code: DATASET_OUTDATED - message: >- - The dataset is outdated. Please try to fetch latest changes - of the dataset and perform the updates - trace: '' - '500': - description: Internal Server Error + code: DATASET_NOT_EXISTS + message: Dataset does not exists with id:telemetry_record-t41 + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '383' + ETag: + schema: + type: string + example: W/"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:53:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -543,578 +817,230 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:41:33+05:30' + ts: '2024-07-16T18:23:16+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 451daa85-9cb8-459a-b681-a256c6e912f0 - responseCode: INTERNAL_SERVER_ERROR + resmsgid: 02fe03f6-c4c4-48f6-9d84-a32cd52f4c13 + responseCode: CONFLICT + result: {} error: - code: DATASET_UPDATE_FAILURE - message: Failed to update dataset - trace: '' + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates /v2/datasets/read/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset get: tags: - - Dataset CRUD APIs - summary: Dataset read + - Dataset api's + summary: Read dataset description: >- This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status - "Live". - This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, - it creates one using the live dataset and returns the dataset details. -
-
- Valid fields that user can request are - "dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date" + "Live". This API accepts the parameter mode=edit to read the draft + dataset. If a draft dataset is not found, it creates one using the live + dataset and returns the dataset details. parameters: - - name: dataset_id - example: sb_telemetry - in: path - required: true - schema: - type: string - - name: status - example: Draft - in: query - required: false - schema: - type: string - - name: fields - example: name - in: query - required: false + - name: Cookie + in: header schema: type: string + example: >- + connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4 - name: mode - example: edit in: query - required: false schema: type: string + example: edit responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '501' + ETag: + schema: + type: string + example: W/"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:08:55 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Read Draft dataset + summary: 'Success: Read live dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:43:58+05:30' + ts: '2024-07-17T17:38:55+05:30' params: status: SUCCESS - resmsgid: d02c643a-d51d-4c67-8a42-9f43e61f459e + resmsgid: 8c8a2852-54bc-43fb-b063-7f359d11930a responseCode: OK result: - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets - entry_topic: local.ingest + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} + redis_db: 54 example-1: - summary: Read Live dataset + summary: 'Success: Read draft dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:44:54+05:30' + ts: '2024-07-17T17:41:00+05:30' params: status: SUCCESS - resmsgid: 3d0efbc5-dea8-4815-a749-b9f9ae16ccfe + resmsgid: 96fd4f42-fa84-4730-bc79-d241a4e335a1 responseCode: OK result: - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts - exclude_fields: [] - entry_topic: local.ingest + timestamp_key: ets + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - version: 1 + file_upload_path: [] + exclude_fields: [] example-2: - summary: Read specific fields from the dataset + summary: 'Success: Read specific column' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:45:44+05:30' + ts: '2024-07-17T17:42:16+05:30' params: status: SUCCESS - resmsgid: da752dfe-0d88-4dd6-a6a8-d858f5960f7e + resmsgid: 02a6b03a-8bf3-4e37-8dcd-859d3e8f904e responseCode: OK result: - name: sb-telemetry - type: dataset - id: sb-telemetry.1 + name: master-test + type: master + id: master-test example-3: - summary: Read version_key from the dataset + summary: 'Success: Read version_key' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:47:48+05:30' + ts: '2024-07-17T17:45:37+05:30' params: status: SUCCESS - resmsgid: a3ad1e17-671f-4294-a834-2ee3f255c9b3 + resmsgid: 805e848a-d260-47c3-b55c-fc9b8323719e responseCode: OK result: - version_key: '1713510527662' + version_key: '1718791650227' example-4: - summary: 'Read Dataset: Create a draft dataset on edit live' + summary: >- + Success: Read from draft, if not present cerate draft from + live dataset and then read value: id: api.datasets.read ver: v2 - ts: '2024-06-18T14:44:54+05:30' + ts: '2024-07-17T17:49:28+05:30' params: status: SUCCESS - resmsgid: 9f8f6a80-f056-424a-97c4-179e2b9199f2 + resmsgid: da70e25b-6ad0-49a7-a39d-340d1d0c46a7 responseCode: OK result: - id: master-telemetry - dataset_id: master-telemetry - type: master-dataset - name: master-telemetry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - eid: - type: string - arrival_format: text - data_type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - arrival_format: text - data_type: string - updatedBy: - type: string - arrival_format: text - data_type: string - lastAccessedOn: - type: string - arrival_format: text - data_type: string - content_id: - type: string - arrival_format: text - data_type: string - oldLastAccessedOn: - type: string - arrival_format: text - data_type: string - progress: - type: number - arrival_format: number - data_type: number - previousProgress: - type: number - arrival_format: number - data_type: number - contentType: - type: string - arrival_format: text - data_type: string - resourceType: - type: string - arrival_format: text - data_type: string - objectId: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - type: - type: string - arrival_format: text - data_type: string - props: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - ver: - type: string - arrival_format: text - data_type: string - syncts: - type: number - arrival_format: number - data_type: number - '@timestamp': - type: string - arrival_format: text - data_type: string - ets: - type: number - arrival_format: number - data_type: number - context: - type: object - properties: - channel: - type: string - arrival_format: text - data_type: string - pdata: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - env: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - flags: - type: object - properties: - ex_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_validation_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_duplicate_skipped: - type: boolean - arrival_format: boolean - data_type: boolean - arrival_format: object - data_type: object - additionalProperties: true - mid: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - additionalProperties: true - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetry - dataset_config: - data_key: ver - timestamp_key: obsrv_meta.syncts - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 70 - file_upload_path: [] - configurations: - indexConfiguration: - index: - Event Arrival Time: obsrv_meta.syncts - rollupSuggestions: {} - processing: - dedupKeys: [] - dropDuplicates: - - 'Yes' - - 'No' - mergedEvent: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - type: - type: string - required: - - id - - type - eid: - type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - updatedBy: - type: string - lastAccessedOn: - type: string - content_id: - type: string - oldLastAccessedOn: - type: string - progress: - type: number - previousProgress: - type: number - contentType: - type: string - resourceType: - type: string - objectId: - type: string - required: - - oldFirstAccessedOn - - updatedBy - - lastAccessedOn - - content_id - - oldLastAccessedOn - - progress - - previousProgress - - contentType - - resourceType - - objectId - type: - type: string - props: - type: string - required: - - data - - type - - props - ver: - type: string - syncts: - type: number - '@timestamp': - type: string - ets: - type: number - context: - type: object - properties: - channel: - type: string - pdata: - type: object - properties: - id: - type: string - required: - - id - env: - type: string - required: - - channel - - pdata - - env - flags: - type: object - properties: - ex_processed: - type: boolean - pp_validation_processed: - type: boolean - pp_duplicate_skipped: - type: boolean - required: - - ex_processed - - pp_validation_processed - - pp_duplicate_skipped - mid: - type: string - type: - type: string - required: - - actor - - eid - - edata - - ver - - syncts - - '@timestamp' - - ets - - context - - flags - - mid - - type + dataset_id: sample1 + name: sample1 + type: event + status: Live tags: [] - created_by: SYSTEM - updated_by: SYSTEM - version_key: '1718702094143' - version: 1 - status: Draft + version: 2 api_version: v2 - transformations_config: [] + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + data_key: '' + timestamp_key: time + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '307' + ETag: + schema: + type: string + example: W/"133-TQ9WpmutsrDcTNkRRmbWOhUChMk" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:20:17 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1122,19 +1048,48 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:03+05:30' + ts: '2024-07-17T17:50:17+05:30' params: status: FAILED - resmsgid: 302fec39-5070-4464-b02f-56b1d0418147 + resmsgid: bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd responseCode: BAD_REQUEST + result: {} error: code: DATASET_INVALID_FIELDS message: >- The specified fields [newname] in the dataset cannot be found. - trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '304' + ETag: + schema: + type: string + example: W/"130-JL/oBB+GUHTrBp278giBHRvO71I" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:21:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1142,43 +1097,26 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:14+05:30' + ts: '2024-07-17T17:51:12+05:30' params: status: FAILED - resmsgid: cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c + resmsgid: 3aad3842-a76e-4fe8-b635-c7fef5f318f9 responseCode: NOT_FOUND + result: {} error: code: DATASET_NOT_FOUND - message: Dataset with the given dataset_id not found - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.read - ver: v2 - ts: '2024-04-19T12:46:34+05:30' - params: - status: FAILED - resmsgid: 29d82020-101d-4506-8fb8-6f3603057064 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_READ_FAILURE - message: Failed to read dataset - trace: '' + message: >- + Dataset with the given dataset_id:new_telemetry_record.1 not + found /v2/datasets/list: post: tags: - - Dataset CRUD APIs - summary: Dataset List + - Dataset api's + summary: Dataset list description: >- This API allows you to list all datasets. User can apply filters on - dataset status and type, sort the datasets as per requested order. + dataset status and type. requestBody: - required: true content: application/json: schema: @@ -1191,70 +1129,65 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: filters: - type: master-dataset + status: + - Live responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '24340' + ETag: + schema: + type: string + example: W/"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:25:36 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset list success when no filters provided + summary: 'Success: Lists all when no filters provided' value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:54:56+05:30' + ts: '2024-07-17T17:55:36+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e7a2c5c1-5aa9-4422-81a5-6f197d3b689e + resmsgid: 97efe04d-e981-493d-9ee7-a6dad6887d64 responseCode: OK result: data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1264,638 +1197,332 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_dropoff_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 - tags: [] + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_dropoff_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: two.1 - dataset_id: two - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713509698978' - created_date: '2024-04-15T02:22:48.270Z' - updated_date: '2024-04-15T02:24:03.308Z' - transformations_config: - - field_key: eid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - field_key: cid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts - exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid + data_key: '' timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} - count: 6 - example-1: - summary: Dataset list success when status filter provided as array - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:55:39+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 836f853f-9ad8-4567-a9bb-74c5d3bba25d - responseCode: OK - result: - data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts + timestamp_key: date exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: taxt_trip + name: taxt_trip + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: obsrv_meta.syncts exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 - tags: [] + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - count: 2 - example-2: - summary: Dataset list success when status filter provided as string - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:56:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 469ef16b-4211-4b71-b063-bbd81cc9df02 - responseCode: OK - result: - data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: telemetry_record-t4 + name: sb-telemetry + type: event + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_events + name: sb-telemetry + type: event + status: Draft + tags: [] + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_record-master + name: sb-telemetry + type: master + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 dataset_config: - data_key: skill + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: generate-schema + name: generate-schema + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' timestamp_key: '' exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + api_version: null dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1905,1453 +1532,6450 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - count: 2 - example-3: - summary: Dataset list success when type filter provided - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:57:28+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 7d9aeb08-2952-485f-9944-4d118da7842e - responseCode: OK - result: - data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + api_version: null dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_pickup_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 1 - example-4: - summary: Dataset list success based on sortBy values + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-test + name: trip-test + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample-trip-details + name: sample-trip-details + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 30 + example-1: + summary: 'Success: Filter based on status as array' value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:58:27+05:30' + ts: '2024-07-17T17:57:38+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 270a4e50-f7e5-4562-ae75-7a967866d065 + resmsgid: 31aba5bc-8492-45ce-be0e-8c52d8716014 responseCode: OK result: data: - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_dropoff_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill + data_key: userid timestamp_key: '' exclude_fields: [] entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 2 - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - status: FAILED - resmsgid: f9e1fae9-4660-409d-bb4d-e29888983d4b - responseCode: BAD_REQUEST - error: - code: DATASET_LIST_INPUT_INVALID - message: >- - #properties/params/additionalProperties should NOT have - additional properties - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:59:15+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 81c3a4a5-e8ac-41de-be00-6d0300ecb106 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_LIST_FAILURE - message: Failed to list dataset - trace: '' - /v2/datasets/diff/{dataset_id}: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + count: 16 + example-2: + summary: 'Success: Filter based on status as string' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T17:59:18+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a08c7ea0-bb1c-4998-b47d-a76e38e87e31 + responseCode: OK + result: + data: + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 8 + example-3: + summary: 'Success: Filter based on dataset type' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:00:41+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 0d1ff2de-42c9-4192-b75d-84f711dbfb55 + responseCode: OK + result: + data: + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + count: 2 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '312' + ETag: + schema: + type: string + example: W/"138-XQplwhrgIYKIg0qtQdRCYWIGTNM" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:32:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:02:26+05:30' + params: + status: FAILED + resmsgid: add9dbe0-f362-4f99-890c-3387c998a049 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_LIST_INPUT_INVALID + message: >- + #properties/params/required must have required property + 'msgid' + /v2/datasets/dataschema: + post: + tags: + - Dataset api's + summary: Data schema generator + description: This api is used to generate data schema for the given dataset event. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + data: + - eid: IMPRESSION + ets: 1672657002221 + ver: '3.0' + mid: IMPRESSION:2b5834e196f485c17c4e49d292af43c0 + actor: + id: 0c45959486f579c24854d40a225d6161 + type: User + context: + channel: '01268904781886259221' + pdata: + id: staging.diksha.portal + ver: 5.1.0 + pid: sunbird-portal + env: public + sid: 23850c90-8a8c-11ed-95d0-276800e1048c + did: 0c45959486f579c24854d40a225d6161 + cdata: [] + rollup: + l1: '01268904781886259221' + uid: anonymous + object: {} + tags: + - '01268904781886259221' + edata: + type: view + pageid: login + subtype: pageexit + uri: >- + https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4 + visits: [] + syncts: 1672657005814 + '@timestamp': '2023-01-02T10:56:45.814Z' + flags: + ex_processed: true + - eid: IMPRESSION + ets: 1672656997928 + ver: '3.0' + mid: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: POST + - url: /v1/org/search + - duration: 0 + - status: OK + - eid: LOG + ets: 1672656998024 + ver: '3.0' + mid: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + type=system, message=EXIT LOG: method : POST, url: + /v1/org/search , For Operation : orgSearch, + params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + errmsg=Invalid value null for parameter hashTagId. + Please provide a valid value., + resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + err=UOS_ORGSER0017, status=FAILED, + responseCode=400}]}} + params: [] + - eid: LOG + ets: 1672657004961 + ver: '3.0' + mid: f34112c7242a3e3a26f0015796b029c2 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: f34112c7242a3e3a26f0015796b029c2 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + ElasticSearchRestHighImpl:search: calling search for + index org_alias, with query = + {"from":0,"size":250,"query":{"bool":{"must":[{"term":{"isTenant.raw":{"value":true,"boost":1.0}}},{"term":{"slug.raw":{"value":"ntp","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":[],"excludes":[]}} + params: [] + - eid: LOG + ets: 1672657006595 + ver: '3.0' + mid: d23ff123-40f0-4262-a69b-b75b46d315a1 + actor: + id: 930a3994-cbe7-4e84-936f-4974096af6f2 + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: d23ff123-40f0-4262-a69b-b75b46d315a1 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, + type=system, message=ENTRY LOG: method : GET, url: + /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 + , For Operation : getUserRolesById, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006611 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, + type=system, message=ENTRY LOG: method : GET, url: + /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , + For Operation : getUserProfileV5, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006620 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user_roles + WHERE userId=?; + params: [] + - eid: LOG + ets: 1672657006645 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: GET + - url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 + - duration: 0 + - status: OK + - eid: LOG + ets: 1672657007238 + ver: '3.0' + mid: d4d34fde-c407-efb6-03bd-9f892ca0f114 + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.portal + pid: learner-service + ver: 5.0.0 + env: User + did: d904c90d9f81ddac20141b94ddd606a0 + cdata: + - id: d4d34fde-c407-efb6-03bd-9f892ca0f114 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user WHERE + id=?; + params: [] + config: + dataset: financial_transactions + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '10760' + ETag: + schema: + type: string + example: W/"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:02:50 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:32:50+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1309aea0-9a97-46e9-bc5e-a16a8a7fb624 + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' format + type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'sid'. The property sid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.sid + - message: >- + The Property 'context.sid' appears to be + 'uuid' format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.context.properties.sid + arrival_format: text + data_type: string + did: + type: string + arrival_format: text + data_type: string + cdata: + type: array + items: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'context.cdata[*].id' + appears to be 'uuid' format type. + advice: >- + Suggest to not to index the high + cardinal columns + resolutionType: DEDUP + severity: LOW + path: >- + properties.context.properties.cdata.items.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: array + data_type: array + rollup: + type: object + properties: + l1: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at + property: 'l1'. The property l1: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System + has updated the property schema to + optional + resolutionType: OPTIONAL + severity: MEDIUM + path: >- + properties.context.properties.rollup.properties.l1 + arrival_format: text + data_type: string + arrival_format: object + data_type: object + uid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uid'. The property uid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.uid + arrival_format: text + data_type: string + arrival_format: object + data_type: object + object: + type: object + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'object'. The property object: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.object + arrival_format: object + data_type: object + tags: + type: array + items: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'tags'. The property tags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.tags + arrival_format: array + data_type: array + edata: + type: object + properties: + type: + type: string + arrival_format: text + data_type: string + pageid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'pageid'. The property pageid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.pageid + arrival_format: text + data_type: string + subtype: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'subtype'. The property subtype: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.subtype + arrival_format: text + data_type: string + uri: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uri'. The property uri: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.uri + - message: >- + The Property 'edata.uri' appears to be 'uri' + format type. + severity: '' + path: properties.edata.properties.uri + arrival_format: text + data_type: string + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'visits'. The property visits: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.visits + arrival_format: array + data_type: array + level: + type: string + arrival_format: text + data_type: string + message: + type: string + arrival_format: text + data_type: string + params: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + syncts: + type: integer + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'syncts'. The property syncts: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.syncts + - message: >- + The Property 'syncts' appears to be 'epoch' format + type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + '@timestamp': + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + '@timestamp'. The property @timestamp: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.@timestamp + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + flags: + type: object + properties: + ex_processed: + type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'ex_processed'. The property ex_processed: + only 1 time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags.properties.ex_processed + arrival_format: boolean + data_type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'flags'. The property flags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + mid: + path: $.mid + cardinality: 67 + index: false + actor.id: + path: $.actor.properties.id + cardinality: 56 + index: false + context.sid: + path: $.context.properties.sid + cardinality: 11 + index: true + edata.uri: + path: $.edata.properties.uri + cardinality: 11 + index: true + context.cdata[*].id: + path: $.context.properties.cdata.items.properties.id + cardinality: 62 + index: false + processing: + dedupKeys: + - mid + - context.cdata[*].id + - actor.id + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:03:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request body' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:33:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bbcc86c2-042d-4f77-bb6e-e1c9116df570 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'data' + example-1: + summary: 'Failure: Invalid request (config not provided)' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:35:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1f856c5e-37f0-41e9-96fb-642471228da2 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'config' + /v2/datasets/status-transition: + post: + tags: + - Dataset api's + summary: Dataset Status Transition + description: >- + This API allows you to perform status transition between 2 states. + Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to + Live, Live to Retired and even Delete a dataset. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: ReadyToPublish + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete success: Deleted dataset successfully' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T12:18:54+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + responseCode: OK + result: + message: Dataset status transition to Delete successful + dataset_id: trip-data + example-1: + summary: 'Live success: Dataset published successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:21:42+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + responseCode: OK + result: + message: Dataset status transition to Live successful + dataset_id: telemetry_record + example-2: + summary: 'ReadyToPublish success: Dataset is ready to publish' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:30:04+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + responseCode: OK + result: + message: Dataset status transition to ReadyToPublish successful + dataset_id: telemetry-events + example-3: + summary: 'Retire success: Dataset retired successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:22:58+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + responseCode: OK + result: + message: Dataset status transition to Retire successful + dataset_id: sb-telemetry + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Live failure: Dataset in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T15:37:43+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + responseCode: BAD_REQUEST + error: + code: DATASET_LIVE_FAILURE + message: >- + Transition failed for dataset: sb-telemetry2 + status:Draft with status transition to Live + trace: '' + example-1: + summary: 'ReadyToPublish failure: Incomplete dataset configs' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:36:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + responseCode: BAD_REQUEST + error: + code: DATASET_CONFIGS_INVALID + message: >- + #properties/denorm_config/properties/denorm_fields/items/required + must have required property 'dataset_name' + trace: '' + example-2: + summary: 'ReadyToPublish failure: Dataset not in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:38:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + responseCode: BAD_REQUEST + error: + code: DATASET_READYTOPUBLISH_FAILURE + message: >- + Transition failed for dataset: telemetry-events + status:Retired with status transition to ReadyToPublish + trace: '' + example-3: + summary: 'Retire Failure: Dataset is already retired' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:42:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 + responseCode: BAD_REQUEST + error: + code: DATASET_RETIRE_FAILURE + message: >- + Transition failed for dataset: master-telemetrry + status:Retired with status transition to Retire + trace: '' + example-4: + summary: >- + Retire failure: Cannot retire master dataset as it is used + by other datasets + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:01:41+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 + responseCode: BAD_REQUEST + error: + code: DATASET_IN_USE + message: >- + Failed to retire dataset as it is in use. Please retire + or delete dependent datasets before retiring this + dataset + trace: '' + example-5: + summary: 'Failure: Invalid request payload provided' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:03:56+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 + responseCode: BAD_REQUEST + error: + code: DATASET_STATUS_INVALID_INPUT + message: >- + #properties/request/properties/status/enum should be + equal to one of the allowed values + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete failure: Dataset not found to delete' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:25:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: master' + trace: '' + example-1: + summary: 'Live failure: Dataset not found to publish' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:35:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: telemetry-data' + trace: '' + example-2: + summary: 'Retire Failure: Dataset not found to retire' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:40:31+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: sb-telemetry2' + trace: '' + /v2/datasets/import: + post: + tags: + - Dataset api's + summary: Dataset import + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + id: sb-telemetry_draft_data + dataset_id: sb-telemetryRPF_draft_data + name: sb-telemetry_draft_data + type: event + extraction_config: + is_batch_event: false + dedup_config: + drop_duplicates: false + dedup_key: id + dedup_period: 604800 + extraction_key: events + validation_config: + validate: true + mode: Strict + dedup_config: + dedup_key: id + drop_duplicates: true + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + syncts: + type: integer + arrival_format: number + data_type: integer + ets: + type: integer + arrival_format: number + data_type: epoch + suggestions: + - message: >- + The Property 'ets' appears to be 'date-time' + format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.ets + flags: + type: object + properties: + ex_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_validation_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_duplicate_skipped: + type: boolean + arrival_format: boolean + data_type: boolean + user_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + device_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + loc_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + content_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + coll_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + arrival_format: object + data_type: object + additionalProperties: false + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + edata: + type: object + properties: + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + type: + type: string + arrival_format: text + data_type: string + duration: + type: number + arrival_format: number + data_type: number + size: + type: integer + arrival_format: number + data_type: integer + query: + type: string + arrival_format: text + data_type: string + filters: + type: object + properties: + objectType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + version: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + status: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + id: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + isRootOrg: + type: boolean + arrival_format: boolean + data_type: boolean + trackable.enabled: + type: string + arrival_format: text + data_type: string + channel: + type: object + arrival_format: object + data_type: object + additionalProperties: false + framework: + type: object + arrival_format: object + data_type: object + additionalProperties: false + resourceType: + type: object + arrival_format: object + data_type: object + additionalProperties: false + identifier: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + contentType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + mimeType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + hashTagId: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + compatibilityLevel: + type: object + properties: + min: + type: integer + arrival_format: number + data_type: integer + max: + type: integer + arrival_format: number + data_type: integer + arrival_format: object + data_type: object + additionalProperties: false + createdBy: + type: string + arrival_format: text + data_type: string + mediaType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + origin: + type: string + arrival_format: text + data_type: string + primaryCategory: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + trackable: + enabled: + type: string + arrival_format: object + data_type: object + additionalProperties: false + sort: + type: object + properties: + lastUpdatedOn: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + topn: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + pageid: + type: string + arrival_format: text + data_type: string + uri: + type: string + arrival_format: text + data_type: string + subtype: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + data: + type: string + arrival_format: text + data_type: string + uaspec: + type: object + properties: + agent: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + system: + type: string + arrival_format: text + data_type: string + platform: + type: string + arrival_format: text + data_type: string + raw: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + state: + type: string + arrival_format: text + data_type: string + props: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + prevstate: + type: string + arrival_format: text + data_type: string + dspec: + type: object + properties: + os: + type: string + arrival_format: text + data_type: string + make: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + idisk: + type: number + arrival_format: number + data_type: number + edisk: + type: number + arrival_format: number + data_type: number + scrn: + type: number + arrival_format: number + data_type: number + camera: + type: string + arrival_format: text + data_type: string + cpu: + type: string + arrival_format: text + data_type: string + sims: + type: integer + arrival_format: number + data_type: integer + webview: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + extra: + type: object + properties: + pos: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + values: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + query: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + mode: + type: string + arrival_format: text + data_type: string + duration: + type: string + arrival_format: object + data_type: object + additionalProperties: false + '@timestamp': + type: string + suggestions: + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + l2: + type: string + arrival_format: text + data_type: string + l3: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + did: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + object: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + version: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + tags: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + additionalProperties: false + denorm_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + denorm_fields: + - denorm_key: actor.id + dataset_id: master-dataset + denorm_out_field: userdata + router_config: + topic: sb-telemetry + dataset_config: + keys_config: + timestamp_key: obsrv_meta.syncts + data_key: '' + partition_key: '' + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + cache_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + redis_db: 0 + file_upload_path: [] + transformations_config: [] + connectors_config: [] + tags: + - tag1 + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1724333643940' + api_version: v2 + sample_data: {} + entry_topic: dev.ingest + created_date: '2024-07-23T18:35:15.690Z' + updated_date: '2024-08-22T13:34:08.041Z' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: overwrite + in: query + schema: + type: boolean + example: 'false' + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '346' + ETag: + schema: + type: string + example: W/"15a-SX5o6plRdfy+akEYhbMwq9zl+oU" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:34:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Fully imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:04:26+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: fd87393e-4b1b-4cf0-8e9a-a0529d691cc3 + responseCode: OK + result: + message: Dataset is imported successfully + data: + id: sb-telemetry_draft_data + version_key: '1732001666671' + example-1: + summary: 'Success: Partially imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T12:59:41+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 379ced0b-eb06-445f-a7e3-8737cb16c351 + responseCode: OK + result: + message: Dataset is partially imported + data: + id: sb-telemetry_draft_data + version_key: '1732001381825' + ignoredFields: + denorm_fields: + - config: + denorm_key: actor.id + denorm_out_field: userdata + dataset_id: master-dataset + details: Master dataset does not exist + example-2: + summary: 'Success: Imported new dataset [overwrite=false]' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:05:32+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1a50e1b1-14b0-4178-aba4-9e09a63d8d1d + responseCode: OK + result: + message: Dataset is imported successfully + data: + id: sb-telemetry_draft + version_key: '1732001732045' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-Dxem7prJM89DOqth+bdZbh0UI1g" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:38:05 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:08:05+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 244318a5-d933-41a5-add6-14f299281069 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_IMPORT_INVALID_CONFIGS + message: >- + #properties/request/required must have required property + 'id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '351' + ETag: + schema: + type: string + example: W/"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:31:11 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:01:11+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1e5b34cd-4473-4439-8cc5-5ff917f47e48 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: >- + Dataset with dataset_id: sb-telemetryRPF_draft_data already + exists. + /v2/datasets/export/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset get: tags: - - Dataset CRUD APIs - summary: Dataset Diff - description: >- - This API allows you to return the difference between the live and draft - dataset by returning additions, deletions and modifications in the - dataset. + - Dataset api's + summary: Dataset export + parameters: + - name: status + in: query + schema: + type: string + example: Live + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '5155' + ETag: + schema: + type: string + example: W/"1423-Tx46QCrIX8So3vSGVkd865IYeEY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:16:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Live Dataset export success' + value: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:46:01+05:30' + params: + status: SUCCESS + resmsgid: ae57c620-2858-40f9-9d8e-33f8f259449d + responseCode: OK + result: + id: content-data-v2 + dataset_id: content-data-v2 + type: master + name: content-data-v2 + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + email: + key: email + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mobile: + key: mobile + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + fare_details: + key: fare_details + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + fare_amount: + key: fare_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + extra: + key: extra + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mta_tax: + key: mta_tax + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tolls_amount: + key: tolls_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + improvement_surcharge: + key: improvement_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + total_amount: + key: total_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + congestion_surcharge: + key: congestion_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + additionalProperties: true + dedup_config: + drop_duplicates: true + dedup_key: tripID + dedup_period: 604800 + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: content-data-v2 + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-2_addedf.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + timestamp_key: tpep_pickup_datetime + data_key: '' + partition_key: '' + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 7 + tags: [] + status: Live + created_by: SYSTEM + updated_by: SYSTEM + data_version: 1 + api_version: v2 + version: 1 + sample_data: + mergedEvent: + tripID: ccbb970c-4d10-4247-be0c-a480996f3429 + VendorID: '1' + tpep_pickup_datetime: '2023-02-26 00:36:10' + tpep_dropoff_datetime: '2023-09-15 00:45:45' + passenger_count: '2' + trip_distance: '1.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '263' + payment_type: '1' + primary_passenger: + email: Jacey_Hintz@yahoo.com + mobile: 247-492-3370 + fare_details: + fare_amount: '8.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.45' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '12.25' + congestion_surcharge: '' + entry_topic: local.ingest + created_date: '2024-10-14T06:23:44.588Z' + updated_date: '2024-10-14T06:23:44.588Z' + example-1: + summary: 'Success: ReadyToPublish Dataset exported successfully' + value: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:48:11+05:30' + params: + status: SUCCESS + resmsgid: 222847a0-32bf-4c63-be68-76f0e51258af + responseCode: OK + result: + id: mydataset + dataset_id: mydataset + name: mydataset + type: event + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + validation_config: + validate: true + mode: Strict + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + email: + key: email + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mobile: + key: mobile + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + additionalProperties: false + fare_details: + key: fare_details + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + fare_amount: + key: fare_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + extra: + key: extra + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + mta_tax: + key: mta_tax + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + tolls_amount: + key: tolls_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + improvement_surcharge: + key: improvement_surcharge + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + total_amount: + key: total_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + congestion_surcharge: + key: congestion_surcharge + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + additionalProperties: false + additionalProperties: false + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: mydataset + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-1 (4) + (1)_5ec855.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 + transformations_config: [] + connectors_config: [] + tags: [] + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1731420131072' + api_version: v2 + sample_data: + mergedEvent: + tripID: 4c77e9d5-538d-4eb7-8db1-4c2c32860aa8 + VendorID: '2' + tpep_pickup_datetime: '2023-11-02 00:18:42' + tpep_dropoff_datetime: '2023-11-02 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Willa67@gmail.com + mobile: 1-720-981-6399 x77055 + fare_details: + fare_amount: 7 + extra: 0.5 + mta_tax: 0.5 + tip_amount: 0 + tolls_amount: 0 + improvement_surcharge: 0.3 + total_amount: 8.3 + congestion_surcharge: 0 + entry_topic: local.ingest + created_date: '2024-11-11T21:31:21.390Z' + updated_date: '2024-11-12T08:32:27.931Z' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '291' + ETag: + schema: + type: string + example: W/"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:20:43 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:50:43+05:30' + params: + status: FAILED + resmsgid: 2097228c-7872-4217-a06a-0ec08e3dc67d + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_EXPORT_FAILURE + message: Dataset with status:Draft cannot be exported + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:15:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:45:10+05:30' + params: + status: FAILED + resmsgid: fd6fdae2-a002-4c15-81e2-3cce539a2a5a + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_FOUND + message: >- + Dataset with the given dataset_id:v1-copy not found to + export + /v2/datasets/copy: + post: + tags: + - Dataset api's + summary: Dataset copy + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + source: + datasetId: dataset-telemetry + isLive: true + destination: + datasetId: bew-copy-live2 parameters: - - name: dataset_id - example: telemetry - in: path - required: true + - name: Content-Type + in: header schema: type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '296' + ETag: + schema: + type: string + example: W/"128-bi4wSfgkyMP13qFE8VPTmgDdgLA" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:26:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - examples: - example-0: - summary: Data Format Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T16:55:25+05:30' - params: - status: SUCCESS - resmsgid: c58aa5f7-f182-4dff-9ee9-16f508e53c51 - responseCode: OK - result: - additions: - - type: dataFormat - items: - - name: extraction_key - value: mid - - name: batch_id - value: id - - name: dedup_config.dedup_key - value: id - modifications: - - type: dataFormat - items: - - name: is_batch_event - value: - from: false - to: true - deletions: [] - example-1: - summary: Timestamp and data key diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:00:44+05:30' - params: - status: SUCCESS - resmsgid: 8ec5fa57-5105-4793-ae96-3a7e0550dbe9 - responseCode: OK - result: - additions: [] - modifications: - - type: timestamp - value: - from: syncts - to: ets - - type: dataKey - value: - from: eid - to: mid - deletions: [] - example-2: - summary: Data Schema Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-28T15:40:48+05:30' - params: - status: SUCCESS - resmsgid: c364416d-b903-4af6-a39b-a4e52c4bf6b5 - responseCode: OK - result: - additions: - - type: dataSchema - items: - - name: eid_1 - value: - arrivalFormat: boolean - dataType: boolean - absolutePath: $.properties.eid_1 - isRequired: false - modifications: - - type: dataSchema - items: - - field: eid - name: type - value: - from: string - to: boolean - - field: eid - name: arrivalFormat - value: - from: text - to: boolean - - field: eid - name: dataType - value: - from: string - to: boolean - deletions: - - type: dataSchema - items: - - name: ver - example-3: - summary: Data Validation Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:05:28+05:30' - params: - status: SUCCESS - resmsgid: ad6b4977-c341-4193-a42f-9b45993e5f8e - responseCode: OK - result: - additions: [] - modifications: - - type: validation - items: - - field: mode - value: - from: IgnoreNewFields - to: Strict - deletions: [] - example-4: - summary: Dataset Dedup diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:07:58+05:30' - params: - status: SUCCESS - resmsgid: e59b841b-e35f-462d-adb1-31018d718f6a - responseCode: OK - result: - additions: - - type: dedup - items: - - name: dedup_key - value: mid - modifications: - - type: dedup - items: - - name: drop_duplicates - value: - from: false - to: true - deletions: [] - example-5: - summary: Dataset Denorm Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:17:35+05:30' - params: - status: SUCCESS - resmsgid: ee558c17-b3a6-41cc-9b07-64105053394e - responseCode: OK - result: - additions: - - type: denorm - items: - - name: syncts - value: - denorm_key: actor.id - dataset_name: mid - modifications: [] - deletions: - - type: denorm - items: - - name: mid - value: - denorm_key: actor.id - dataset_name: mid - example-6: - summary: Dataset transformation diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:25:12+05:30' - params: - status: SUCCESS - resmsgid: f4389891-9c09-4ccd-affe-39970590045b - responseCode: OK - result: - additions: - - type: transformations - items: - - name: skill_id - value: - transformation_function: - type: mask - condition: null - expr: skill_id - mode: Strict - metadata: - section: pii - modifications: - - type: transformations - items: - - name: new - field: transformation_function.type - value: - from: jsonata - to: encrypt - metadata: - section: transformation - deletions: - - type: transformations - items: - - name: mid - value: - metadata: - section: transformation - - name: name - value: - metadata: - section: transformation - - name: neew - value: - metadata: - section: additionalFields - /v2/files/generate-url: + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:56:12+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 28072631-8e80-45eb-906f-d933a90646d0 + responseCode: OK + result: + dataset_id: new-copy-live2 + message: Dataset clone successful + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '328' + ETag: + schema: + type: string + example: W/"148-jUDshghYf/jOhmmVkvtHlfJT864" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:22:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:52:41+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: f2fd51bb-fc18-4278-a229-47fa25398e69 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_EXISTS + message: Dataset dataset-telemetry does not exists + /v2/connectors/list: post: tags: - - Dataset CRUD APIs - summary: Files generate URL - description: >- - This API allows you to generate pre signed urls from cloud providers for - getting and putting files. On access: write, upload urls are generated to put files and on access: read, download urls are generated to get files. + - Connector api's + summary: Connector list requestBody: - content: - application/json: - examples: - example1: - summary: Generate upload URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: write - example2: - summary: Generate download URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: read + content: + application/json: + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Draft responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '250' + ETag: + schema: + type: string + example: W/"fa-+eWKIfUxsWBGuJy23qSucgLXke4" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:55:51 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Success: Generate Put URL' + summary: 'Success: Filtered based on status' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:04:15+05:30' + ts: '2024-07-30T15:25:51+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a79f1260-1b34-4b96-9c12-cf0f5600e9f3 + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a responseCode: OK result: - - filePath: >- - obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=0zWi7R9wqUT6uW0QuN%2F0nyvaloSCqRPTvzHn%2BpkCWdk%3D - - filePath: >- - obsrv-onest/api-service/user_uploads/school_data_14092a.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data_14092a.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=35g%2F46r%2FOI490%2B8pKry9mEbAgXOXznmCyw7L%2FSLITM0%3D + data: [] + count: 0 example-1: - summary: 'Success: Generate Get URL' + summary: 'Success: Filtered based on category' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:05:47+05:30' + ts: '2024-07-31T18:55:03+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 141717fb-9b94-463d-b0f1-1bab40b187cc + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab responseCode: OK result: - - filePath: obsrv-onest/api-service/user_uploads/telemetry.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=UEWKMP9wcETNKA1sXo%2BVI6qZ7Iqk%2BJg6kglaEXz7%2B1Y%3D - - filePath: obsrv-onest/api-service/user_uploads/school_data.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=jXzootTAoUAWLYKF9AYwKh9Mzl%2FVctlebTBHVz%2BbaKM%3D - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Invalid access value requested' + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + count: 4 + example-2: + summary: 'Success: Connectors list with all filter options' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:08:23+05:30' + ts: '2024-07-31T18:56:32+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b1da48a0-5381-49a1-9d5f-ce6c7c65fdcf - responseCode: BAD_REQUEST - error: - code: FILES_GENERATE_URL_INPUT_INVALID - message: >- - #properties/request/properties/access/enum should be - equal to one of the allowed values - trace: '' - example-1: - summary: >- - Failure: Number of file url requested exceeds the limit - configured + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 11a2f537-bd98-405b-97e5-0f0d5b86b2c3 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-3: + summary: 'Success: Connectors list without filters' value: - id: api.files.generate-url + id: api.connectors.list ver: v2 - ts: '2024-05-30T12:10:29+05:30' + ts: '2024-07-31T18:57:37+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5828abd2-2934-40ef-8fa1-a9bca1e7e24e - responseCode: BAD_REQUEST - error: - code: FILES_URL_GENERATION_LIMIT_EXCEED - message: 'Pre-signed URL generation failed: limit exceeded.' - trace: '' - /v2/obsrv/data/sql-query: - post: - tags: - - Dataset CRUD APIs - summary: SQL Wrapper - description: This api is a wrapper api, used to query the data. - requestBody: - content: - application/json: - schema: - type: object - example: - query: >- - SELECT "channel","flags", COUNT(*) AS "Count" FROM "wikipedia" - WHERE "page" IS NOT NULL GROUP BY 1, 2 ORDER BY 3 DESC - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - example: - - channel: '#en.wikipedia' - flags: '' - Count: 4776 - - channel: '#fr.wikipedia' - flags: '!M' - Count: 31 - - channel: '#fa.wikipedia' - flags: B - Count: 30 - - channel: '#vi.wikipedia' - flags: 'N' - Count: 9 - - channel: '#ar.wikipedia' - flags: '!N' - Count: 8 - - channel: '#da.wikipedia' - flags: '!' - Count: 8 - - channel: '#hy.wikipedia' - flags: '!N' - Count: 8 - - channel: '#id.wikipedia' - flags: '!N' - Count: 8 - - channel: '#eu.wikipedia' - flags: M - Count: 3 - - channel: '#fa.wikipedia' - flags: '!N' - Count: 3 - - channel: '#fi.wikipedia' - flags: 'N' - Count: 3 - - channel: '#he.wikipedia' - flags: NB - Count: 3 - - channel: '#hi.wikipedia' - flags: '' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!N' - Count: 1 + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c2467e01-0a2d-401c-aa3d-dd16b804f723 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 '400': description: Bad Request - content: - application/json: + headers: + X-Powered-By: schema: - type: object - example: - id: api.obsrv.data.sql-query - ver: v2 - ts: '2024-05-30T16:31:57+05:30' - params: - status: FAILED - resmsgid: 7709ef5b-d7f3-48a5-bf6c-cf243fb6d2eb - responseCode: BAD_REQUEST - error: - code: SQL_QUERY_EMPTY_REQUEST - message: Failed to query. Invalid request - trace: '' - '500': - description: Internal Server Error + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '399' + ETag: + schema: + type: string + example: W/"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:43:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object example: - id: api.obsrv.data.sql-query + id: api.connectors.list ver: v2 - ts: '2024-05-29T13:03:44+05:30' + ts: '2024-07-30T15:13:14+05:30' params: status: FAILED - resmsgid: 0b4b7168-2b70-406b-84ce-c320b65559eb - responseCode: INTERNAL_SERVER_ERROR + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fadde0-8c59-4420-8ab3-56474b01670b + responseCode: BAD_REQUEST + result: {} error: - code: ERR_BAD_RESPONSE - message: Request failed with status code 500 - trace: '' - /v2/datasets/status-transition: - post: + code: CONNECTORS_LIST_INPUT_INVALID + message: >- + #properties/request/properties/filters/properties/status/minItems + must NOT have fewer than 1 items + /v2/connectors/read/{connector_id}: + parameters: + - name: connector_id + in: path + required: true + schema: + type: string + description: Unique identifier for the connector + get: tags: - - Dataset CRUD APIs - summary: Dataset Status Transition - description: >- - This API allows you to perform status transition between 2 states. - Allowed status transition are: -
- - Draft to ReadyToPublish when the dataset configuration is complete and configuration is validated, ReadyToPublish to Live to make dataset live and make it actively running within the pipeline, Live to Retired when dataset is needed to be stopped and retired.User can even Delete the dataset of Draft and ReadyToPublish status. - requestBody: - content: - application/json: - examples: - example1: - summary: Transition to ReadyToPublish - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: ReadyToPublish - example2: - summary: Transition to Live - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Live - example3: - summary: Transition to Retired - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Retire - example4: - summary: Delete a draft dataset - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Delete + - Connector api's + summary: Connector Read responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '3304' + ETag: + schema: + type: string + example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 12:47:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete success: Deleted dataset successfully' + summary: 'Success: Read live Connector' value: - id: api.datasets.status + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:18:54+05:30' + ts: '2024-07-31T18:17:54+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced responseCode: OK result: - message: Dataset deleted successfully - dataset_id: master.1 + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: + type: object + properties: + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: + type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly + fieldDescription: + - type: string + description: '' + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: + batch_size: + ui:readonly: true + max_batches: + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' example-1: - summary: 'Live success: Dataset published successfully' + summary: 'Success: Read Draft Connector' value: - id: api.datasets.status + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:21:42+05:30' + ts: '2024-08-01T12:47:12+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + resmsgid: b6fcfb05-246c-4a1b-9eb1-27497ee9b80b responseCode: OK result: - message: Dataset published successfully - dataset_id: telemetry - example-2: - summary: 'ReadyToPublish success: Dataset is ready to publish' + id: mssql-connector-2.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 2.0.0 + description: >- + The MS SQL Connector is used to move data from any MS + SQL Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Draft + ui_spec: {} + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '276' + ETag: + schema: + type: string + example: W/"114-izVC8DsHdeSfau/USVJvnIqZIMQ" + Date: + schema: + type: string + example: Thu, 01 Aug 2024 09:32:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T15:02:48+05:30' + params: + status: FAILED + resmsgid: 712e7298-99f8-4694-9011-4232fcfd664a + responseCode: NOT_FOUND + result: {} + error: + code: CONNECTOR_NOT_FOUND + message: 'Connector not found: postgres-conn' + /v2/data/in/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset + post: + tags: + - Data Ingest + summary: Data ingest + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + data: + eid: INTERACT + date: '2022-01-01' + ver: '3.0' + syncts: 1668591949682 + ets: 1668591949682 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '261' + ETag: + schema: + type: string + example: W/"105-UtaCmh0qZMBRBdniNq74Gr+4YAo" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:05:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Data ingested successfully' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:30:04+05:30' + ts: '2024-05-30T23:35:48+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 9a44ac5b-ef82-46f7-92c5-c5b39ef88764 responseCode: OK result: - message: Dataset status transition to ReadyToPublish successful - dataset_id: telemetry-events - example-3: - summary: 'Retire success: Dataset retired successfully' + message: Data ingested successfully + example-1: + summary: 'Success: Data ingested successfully (batch)' value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T12:22:58+05:30' + ts: '2024-05-30T23:36:49+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: a220041c-0c28-415a-9687-9fb2e211f8c4 responseCode: OK result: - message: Dataset retired successfully - dataset_id: sb-telemetry + message: Data ingested successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '300' + ETag: + schema: + type: string + example: W/"12c-GNCIUUDxOZD3UfM311sjnmFIgPc" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:03:32 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '2024-05-30T23:33:32+05:30' + params: + status: FAILED + resmsgid: acf07609-77de-4ab5-81ea-42e41b8b95ff + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_INGESTION_INVALID_INPUT + message: '#required should have required property ''id''' + trace: '' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '316' + ETag: + schema: + type: string + example: W/"13c-O9iirC/EyneYXQzth7iwEEy1UV4" + Date: + schema: + type: string + example: Thu, 18 Apr 2024 10:17:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Live failure: Dataset in draft state' + summary: Entry topic not found value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T15:37:43+05:30' + ts: '2024-04-18T15:47:00+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 11c32a3a-fdeb-4e00-a9cf-f6433ade5608 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_NOT_READY_FOR_PUBLISH - message: Failed to publish dataset as it is in draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' example-1: - summary: 'ReadyToPublish failure: Incomplete dataset configs' + summary: 'Failure: Dataset not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:36:16+05:30' + ts: '2024-05-30T23:37:29+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: d4db36b4-37b0-4170-a4cf-9d2ae8fa0416 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_CONFIGS_INVALID - message: >- - #properties/denorm_config/properties/denorm_fields/items/required - must have required property 'dataset_name' + code: DATASET_NOT_FOUND + message: Dataset with id not found trace: '' example-2: - summary: 'ReadyToPublish failure: Dataset not in draft state' + summary: 'Failure: Entry topic not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:38:14+05:30' + ts: '2024-05-30T23:39:26+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 62309380-3e83-442f-877e-359ed2774bbf responseCode: BAD_REQUEST + result: {} error: - code: DATASET_READYTOPUBLISH_FAILURE - message: >- - Failed to mark dataset Ready to publish as it not in - draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' - example-3: - summary: 'Retire Failure: Dataset is already retired' + /v2/data/query/test: + post: + tags: + - Data query + summary: Data query + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + context: + aggregationLevel: day + query: >- + Select * from "check" WHERE __time >= timestamp '2020-12-31' + AND __time < TIMESTAMP '2024-01-21' Limit 1 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '313' + ETag: + schema: + type: string + example: W/"139-De+IthAwrGNR+J11CwlNf5RSMmw" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:17:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: native query (interval as array type)' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:42:18+05:30' + ts: '2024-05-30T23:47:45+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_RETIRED - message: Dataset is already retired - trace: '' - example-4: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 134efe35-c096-4cab-ad14-db6a8952f264 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-1: + summary: 'Success: native query (queryType: scan)' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:51:14+05:30' + params: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 4fc6e3c1-4715-47b7-9137-7713fb2fbe72 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-2: summary: >- - Retire failure: Cannot retire master dataset as it is used - by other datasets + Success: native query (normal intervals & queryType : + timeseries) value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:01:41+05:30' + ts: '2024-05-30T23:51:49+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 - responseCode: BAD_REQUEST - error: - code: DATASET_IN_USE - message: Failed to retire dataset as it is used by other datasets - trace: '' - example-5: - summary: 'Failure: Invalid request payload provided' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75d7ae48-a3db-4367-b50f-34eb99ac3480 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + school_id: 0 + example-3: + summary: 'Success: sql query' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:03:56+05:30' + ts: '2024-05-30T23:42:12+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 - responseCode: BAD_REQUEST - error: - code: DATASET_STATUS_INVALID_INPUT - message: >- - #properties/request/properties/status/enum should be - equal to one of the allowed values - trace: '' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 2c981011-76da-3000-97f3-eafac939e59f + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '371' + ETag: + schema: + type: string + example: W/"173-OP8NcbSqLKFO92PIyUmMk0lNsXs" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:16:29 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:46:29+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: b6434700-dd92-4f64-9250-a22939e753b9 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_OUT_INVALID_DATE_RANGE + message: >- + Invalid date range! make sure your range cannot be more than + 30 days + trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '356' + ETag: + schema: + type: string + example: W/"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:13:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete failure: Dataset not found to delete' + summary: 'Failure: Datasource not found in live table' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T12:25:36+05:30' + ts: '2024-05-30T23:43:12+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: f922f120-2aea-49af-9a76-7312fe2eb266 responseCode: NOT_FOUND + result: {} error: - code: DATASET_NOT_FOUND - message: Dataset not found to delete + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Datasource telemetry-eventssss not available for + querying trace: '' example-1: - summary: 'Live failure: Dataset not found to publish' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T15:35:59+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c - responseCode: NOT_FOUND - error: - code: DATASET_NOT_FOUND - message: Dataset not found to publish - trace: '' - example-2: - summary: 'Retire Failure: Dataset not found to retire' + summary: 'Failure: Datasource not found in druid' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:40:31+05:30' + ts: '2024-05-30T23:50:27+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75bd4313-d504-4fd3-92ab-ee2a685beb83 responseCode: NOT_FOUND - error: - code: DATASET_NOT_FOUND - message: Dataset not found to retire - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T16:24:12+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 92928434-719f-47d4-9946-1e40ecd53253 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_STATUS_FAILURE - message: Failed to perform status transition on datasets - trace: '' - /v2/data/in/{datasetId}: - post: - tags: - - Data Ingest - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - summary: Data ingest - description: >- - This API allows you to ingest events to a data streaming platform in individual or batch format. - requestBody: - content: - application/json: - examples: - example1: - summary: Ingest individual event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: - eid: INTERACT - date: '2022-01-01' - ver: '3.0' - syncts: 1668591949682 - ets: 1668591949682 - example2: - summary: Ingest batch event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: [{ "eid": "INTERACT","date": "2022-01-01","ver": "3.0","syncts": 1668591949682}] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Data ingest successful(individual) - value: - id: api.data.in - ver: v2 - ts: 1711966306164 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - example-1: - summary: Data ingest successfully(batch) - value: - id: api.data.in - ver: v2 - ts: 1711966352987 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.data.in - ver: v2 - ts: 1711966410766 - params: - status: FAILED - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATA_INGESTION_INVALID_INPUT - message: "#required should have required property 'id'" - trace: '' - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Dataset not found - value: - id: api.data.in - ver: v2 - ts: 1711966446364 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_FOUND - message: Dataset with id not found - trace: '' - example-1: - summary: Entry topic not found - value: - id: api.data.in - ver: v2 - ts: 1711975482739 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: ENTRY_TOPIC_NOT_FOUND - message: Entry topic is not defined - trace: '' - /v2/data/query/{datasetId}: - post: - tags: - - Data query - summary: Data query - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - description: >- - This API allows you to query your datasource. - requestBody: - content: - application/json: - examples: - example1: - summary: SQL query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: >- - SELECT * FROM "rollups-configured" WHERE __time >= TIMESTAMP - '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1 - example2: - summary: Native query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - mid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: - quertType: scan - dataSource: rollups-configured - intervals: - type: intervals - intervals: ["2022-01-01/2022-02-01"] - granularity: day - columns: ["eid","__time"] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: sql query' - value: - id: api.data.out - ver: v2 - ts: 1711971972855 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - - __time: '2022-01-01T00:00:00.000Z' - eid: '1' - ver: '3.0' - flags_ex_processed: 'true' - flags_pp_validation_processed: 'true' - flags_pp_duplicate_skipped: 'true' - flags_device_denorm: 'true' - flags_user_denorm: 'true' - flags_loc_denorm: 'true' - derivedlocationdata_district: AGRA - derivedlocationdata_from: user-profile - derivedlocationdata_state: Uttar Pradesh - mid: 6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0 - type: events - actor_type: User - actor_id: 311663b2-d7de-4d46-8803-20407eaa3403 - edata_type: session - userdata_subject: null - userdata_district: AGRA - userdata_usersubtype: hm - userdata_grade: null - userdata_usersignintype: Self-Signed-In - userdata_usertype: administrator - userdata_userlogintype: administrator - userdata_state: Uttar Pradesh - devicedata_statecustomcode: '29' - devicedata_country: India - devicedata_iso3166statecode: IN-KA - devicedata_city: Bengaluru - devicedata_countrycode: IN - devicedata_state: Karnataka - devicedata_devicespec_idisk: '106.47' - devicedata_devicespec_webview: 107.0.5304.105 - devicedata_devicespec_os: Android 12 - devicedata_devicespec_scrn: '6.53' - devicedata_devicespec_sims: '-1' - devicedata_devicespec_cpu: "abi: arm64-v8a processor\t: 0 " - devicedata_devicespec_id: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - devicedata_devicespec_camera: '' - devicedata_devicespec_edisk: '106.27' - devicedata_devicespec_make: vivo 1915 - devicedata_statecode: KA - devicedata_districtcustom: BENGALURU URBAN SOUTH - devicedata_statecustomname: Karnataka - devicedata_userdeclared_district: AGRA - devicedata_userdeclared_state: Uttar Pradesh - context_cdata_id: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_cdata_type: UserSession - context_env: sdk - context_channel: '0126796199493140480' - context_pdata_id: preprod.diksha.app - context_pdata_pid: sunbird.app - context_pdata_ver: 4.10.1023preproduction - context_sid: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_did: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - context_rollup_l1: '0126796199493140480' - object_id: '' - object_type: '' - object_version: '' - object_rollup: '{}' - count: 3 - edata_duration_sum: null - example-1: - summary: 'Success: native query' - value: - id: api.data.out - ver: v2 - ts: 1711973520793 - params: - status: SUCCESS - errmsg: '' - responseCode: OK - result: - - timestamp: '2022-01-01T00:00:00.000Z' - result: - mid: 1 - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: invalid date range (sql)' - value: - id: api.data.out - ver: v2 - ts: 1711972193426 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - example-1: - summary: 'Failure: Invalid date range (native)' - value: - id: api.data.out - ver: v2 - ts: 1711972501847 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.data.out - ver: v2 - ts: 1711972085550 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.data.out - ver: v2 - ts: 1711975034302 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-2: - summary: 'Failure: Datasource not found in live table (native)' - value: - id: api.data.out - ver: v2 - ts: 1711974978635 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" - example-3: - summary: 'Failure: Datasource not found in druid(native)' - value: - id: api.data.out - ver: v2 - ts: 1711972569115 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-2: + summary: 'SQL Failure: dataset not found in druid' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:52:26+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 19849769-c290-4599-a79e-862dcbce1124 + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-3: + summary: 'SQL Failure: Datasource not found in live table' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:53:02+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 530a12f9-04f3-4986-8f52-b3184b9194bd + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource undefined not available for querying + trace: '' /v2/template/read/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template get: tags: - Query Templates summary: Read template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '537' + ETag: + schema: + type: string + example: W/"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:24:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3362,10 +7986,10 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:09:55+05:30' + ts: '2024-05-30T23:54:14+05:30' params: status: SUCCESS - resmsgid: 8ef3a9d0-8658-47dd-b6cb-a4b1777a1ea4 + resmsgid: ad1c0a98-1ba3-4203-9fac-59ccaf442347 responseCode: OK result: template_id: jsontemplate111 @@ -3382,24 +8006,54 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:10:54+05:30' + ts: '2024-05-30T23:54:38+05:30' params: status: SUCCESS - resmsgid: 656185ef-527d-4b60-ac19-263da85142b8 + resmsgid: 70047866-a4b4-4fce-b9c4-4699fcab2dc6 responseCode: OK result: template_id: sql1 template_name: sql1 query: >- - "SELECT * FROM {{DATASET}} WHERE __time BETWEEN - TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP + {{ENDDATE}} lIMIT {{LIMIT}}" query_type: sql created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-04-29T11:29:58.759Z' - updated_date: '2024-04-29T11:29:58.759Z' + created_date: '2024-05-13T07:29:04.117Z' + updated_date: '2024-05-13T07:29:04.117Z' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '294' + ETag: + schema: + type: string + example: W/"126-zq5j1yor+xr1XFEEbP09fQTDI/k" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:25:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3407,30 +8061,60 @@ paths: example: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:11:45+05:30' + ts: '2024-05-30T23:55:16+05:30' params: status: FAILED - resmsgid: bc6a4621-4ac5-4959-b467-1134ea874960 + resmsgid: 958bf0e7-bdb8-4153-abdc-ab84e8004a0e responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template sql100 does not exists trace: '' /v2/template/delete/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template delete: tags: - Query Templates summary: Delete template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '241' + ETag: + schema: + type: string + example: W/"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo" + Date: + schema: + type: string + example: Tue, 30 Apr 2024 06:28:27 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3447,6 +8131,35 @@ paths: message: Template yashash-k deleted successfully '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '303' + ETag: + schema: + type: string + example: W/"12f-99pWw8VTwuVfDAhinC55JXfNyyg" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:28:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3454,27 +8167,21 @@ paths: example: id: api.query.template.delete ver: v2 - ts: '2024-04-29T18:29:09+05:30' + ts: '2024-05-30T23:58:41+05:30' params: status: FAILED - resmsgid: 6c62ff4d-0a63-4384-a365-7bbef951f288 + resmsgid: 9b7f81fb-6705-4d32-9bd3-139cd5a211b9 responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template json_template does not exists trace: '' - /v2/template/create/{template_id}: + /v2/template/create: post: tags: - Query Templates summary: Create query template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path requestBody: content: application/json: @@ -3493,9 +8200,44 @@ paths: datasetId: '{{DATASET}}' intervals: '{{STARTDATE}}/{{ENDDATE}}' limit: '{{LIMITS}}' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '359' + ETag: + schema: + type: string + example: W/"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 16:59:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3533,6 +8275,35 @@ paths: message: The query template has been saved successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '407' + ETag: + schema: + type: string + example: W/"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:05:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3549,6 +8320,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 62e18342-7e25-4122-8fca-6fb12fac3ff0 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID message: >- @@ -3566,6 +8338,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID_INPUT message: >- @@ -3574,6 +8347,35 @@ paths: trace: '' '409': description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '350' + ETag: + schema: + type: string + example: W/"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:03:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3587,6 +8389,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 18b6b123-4df5-4124-b6ec-73b667250e1c responseCode: CONFLICT + result: {} error: code: QUERY_TEMPLATE_ALREADY_EXISTS message: Template josnaks-aaa already exists @@ -3616,6 +8419,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '6864' + ETag: + schema: + type: string + example: W/"1ad0-xp24UiXXXiFWplmv5Acja7prSYM" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 13:46:03 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4176,17 +9008,17 @@ paths: created_date: '2024-04-25T06:31:20.819Z' updated_date: '2024-04-25T06:31:20.819Z' /v2/template/update/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template patch: tags: - Query Templates summary: update template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path requestBody: content: application/json: @@ -4207,6 +9039,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '314' + ETag: + schema: + type: string + example: W/"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:51:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4214,341 +9075,929 @@ paths: example: id: api.query.template.update ver: v2 - ts: '2024-05-02T13:16:31+05:30' + ts: '2024-05-10T11:21:47+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ab869754-d2f0-4b9a-b38b-04d9e5901b5d + resmsgid: 9e4a6959-0eb9-4fc4-8e6f-2eea534d1384 responseCode: OK result: message: Query template updated successfully templateId: sql11template1 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '429' + ETag: + schema: + type: string + example: W/"1ad-5sb8WUekFL8s4c1Ink6bUByoHho" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:53:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: required variables not exists to update' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:23:54+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 66b95cb3-2ef2-4735-9045-2674da552dbd + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Invalid template provided, A template should consist of + variables DATASET,STARTDATE,ENDDATE and type of json,sql + trace: '' + example-1: + summary: 'Failure: Template name validation failure' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:26:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: d6428fcf-53c9-465d-9431-769218f775b8 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Template name should contain alphanumeric characters and + single space between characters + trace: '' + example-2: + summary: 'Failure: query_type should when updating query' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:32:57+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c7a8675a-73f2-4764-abba-bfdf9f8b4621 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query_type when property query is present + trace: '' + example-3: + summary: 'Failure: query should present when updating query_type' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:34:55+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 2c1098b2-d7b3-4d39-98ee-e3e790fd23b4 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query when property query_type is present + trace: '' + /v2/template/query/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template + post: + tags: + - Query Templates + summary: query template + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.query + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + startdate: '2020-12-31' + enddate: '2024-12-31' + aggregationLevel: month + dataset: test + limit: 5 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-9hKB38iHEwYPT2MgF8puXcq05Ew" + Date: + schema: + type: string + example: Tue, 14 May 2024 06:22:24 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: JSON template with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:52:24+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + responseCode: OK + result: + - timestamp: '2023-09-01T00:00:00.000Z' + result: + school_id: 0 + example-1: + summary: 'Success: SQL template query with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:33:06+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '336' + ETag: + schema: + type: string + example: W/"150-T/XeSIt7PR7GcGEbET1e8n9zX7k" + Date: + schema: + type: string + example: Thu, 02 May 2024 07:29:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: query should present when query_type provided' + summary: 'Failure: invalid date range (native template)' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:30:36+05:30' + ts: '2024-05-02T12:59:14+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: f11f5287-77cd-43f0-b7f6-6d6b94b9cea2 + resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 responseCode: BAD_REQUEST + result: {} error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - #properties/request/dependencies should have property - query when property query_type is present + Invalid date range! make sure your range cannot be more + than 30 days trace: '' example-1: - summary: 'Failure: query_type should present when query is given' - value: - id: api.query.template.update - ver: v2 - ts: '2024-05-02T11:33:23+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 937cf790-867b-450b-afc9-86a96d02f636 - responseCode: BAD_REQUEST - error: - code: QUERY_TEMPLATE_INVALID_INPUT - message: >- - #properties/request/dependencies should have property - query_type when property query is present - trace: '' - example-2: - summary: 'Failure: template name validation failure' + summary: 'Failure: invalid date range' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:34:18+05:30' + ts: '2024-05-13T13:28:18+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9f3ae5aa-2e98-4204-8f87-b11e5bb3865a + resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 responseCode: BAD_REQUEST error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - Template name should contain alphanumeric characters and - single space between characters + Invalid date range! make sure your range cannot be more + than 30 days trace: '' - example-3: - summary: 'Failure: required template_id' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '357' + ETag: + schema: + type: string + example: W/"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U" + Date: + schema: + type: string + example: Mon, 13 May 2024 07:51:46 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Datasource not found in druid' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:38:16+05:30' + ts: '2024-05-13T13:21:46+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 94b71f80-bd1e-4283-8f3d-83833b60d08c - responseCode: BAD_REQUEST + resmsgid: b35a7050-b94c-4944-9630-233c9542272e + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_SOURCE_NOT_FOUND message: >- - #properties/request/required should have required - property 'template_id' + Dataset test with table hour is not available for + querying trace: '' - example-4: - summary: 'Failure: required variables not exists' + example-1: + summary: 'Failure: Datasource not found in live table' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T13:13:22+05:30' + ts: '2024-05-13T13:23:47+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dff3c22a-c198-4d1c-b87a-07d725f868f7 - responseCode: BAD_REQUEST + resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID - message: >- - Invalid template provided, A template should consist of - variables DATASET,STARTDATE,ENDDATE and type of json,sql + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource test11 not available for querying trace: '' - /v2/template/query/{template_id}: + /alerts/v1/notifications/search: + post: + tags: + - Alert Notification-channels + summary: search channels + requestBody: + content: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/create: + post: + tags: + - Alert Notification-channels + summary: Add channel (Email) + requestBody: + content: + application/json: + schema: + type: object + example: + manager: grafana + name: functional-metrics-email + type: email + config: + recipientAddresses: yravinderkumar33@gmail.com;ravinder@sanketika.in + subject: Obsrv Prod Alert + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert Notification-channels + summary: publish channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/test: + post: + tags: + - Alert Notification-channels + summary: test channel + requestBody: + content: + application/json: + schema: + type: object + example: + message: Testing Email integration. If you can read this, it's working! + payload: + error: + selectChannel: true + configureChannel: true + manager: grafana + name: udhw + type: email + config: + recipientAddresses: jerald@sanketika.in + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Notification-channels + summary: update channel + requestBody: + content: + application/json: + schema: + type: object + example: + name: updated name + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Notification-channels + summary: delete channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/create: + post: + tags: + - Alert silence + summary: Add Silence + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2024-01-11T07:56:23Z' + endDate: '2024-01-11T08:56:23Z' + alertId: c7464d32-1d8d-4eaf-9b23-1313a3ff8149 + manager: grafana + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert silence + summary: Get Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert silence + summary: Delete Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/search: + get: + tags: + - Alert silence + summary: Fetch All Silences + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert silence + summary: Edit Request + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2023-08-09T10:50:59Z' + endDate: '2023-08-10T10:30:59Z' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alerts Wrapper + summary: 'delete rule ' + description: This URLwill provided access to user to delete any custom rule. + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alerts Wrapper + summary: 'edit rule ' + description: >- + This URL will provide access to users to edit any properties in the + Alert rule and save the changes. + requestBody: + content: + application/json: + schema: + type: object + example: + description: >- + This alert rule is designed to promptly notify you when one or + more servers in your infrastructure become unresponsive or + inaccessible. + labels: + severity: warning + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'publish rule ' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/create: + post: + tags: + - Alerts Wrapper + summary: 'Add Rule ' + requestBody: + content: + '*/*': + schema: + type: string + example: >- + "{\n \"name\": \"Total Api Calls1\",\n + \"manager\":\"grafana\",\n \"description\": \"This alert is + set up to notify you when the CPU usage on a host reaches a Low + level.\",\n \"expression\": \"(node_total_api_calls) > + 20\",\n \"category\": \"Infra\",\n + \"severity\":\"warning\",\n \"frequency\": \"1m\",\n + \"interval\": \"1m\",\n \"labels\": {\n \"component\": + \"api\",\n \"notificationChannel\": \"slack\"\n + },\n \"annotations\":{\n \"summary\":\"Host Low CPU + usage\"\n },\n \"metadata\": {\n \"query\": + [\n {\n \"refId\": + \"A\",\n \"datasourceUid\": + \"$datasourceUid\",\n \"queryType\": + \"\",\n \"relativeTimeRange\": + {\n \"from\": 600,\n + \"to\": 0\n },\n \"model\": + {\n \"refId\": \"A\",\n + \"hide\": false,\n \"editorMode\": + \"code\",\n \"expr\": + \"node_total_api_calls\",\n \"legendFormat\": + \"__auto\",\n \"range\": + true\n }\n },\n + {\n \"refId\": \"B\",\n + \"datasourceUid\": \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"B\",\n \"hide\": + false,\n \"type\": + \"reduce\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"B\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"reducer\": + \"last\",\n \"expression\": + \"A\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n },\n {\n \"refId\": + \"C\",\n \"datasourceUid\": + \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"C\",\n \"hide\": + false,\n \"type\": + \"threshold\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [\n + 20\n + ],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"C\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"expression\": + \"B\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n }\n ]\n }\n}" + parameters: + - name: Accept + in: header + schema: + type: string + example: application/json, text/plain, */* + - name: Accept-Language + in: header + schema: + type: string + example: en-GB,en + - name: Cache-Control + in: header + schema: + type: string + example: no-store + - name: Connection + in: header + schema: + type: string + example: keep-alive + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; + grafana_session=4eb514f6fef4ad6884e47e50254af650; + grafana_session_expiry=1687239735 + - name: Origin + in: header + schema: + type: string + example: http://localhost:3001 + - name: Pragma + in: header + schema: + type: string + example: no-store + - name: Referer + in: header + schema: + type: string + example: http://localhost:3001/alertRules/add + - name: Sec-Fetch-Dest + in: header + schema: + type: string + example: empty + - name: Sec-Fetch-Mode + in: header + schema: + type: string + example: cors + - name: Sec-Fetch-Site + in: header + schema: + type: string + example: same-origin + - name: Sec-GPC + in: header + schema: + type: integer + example: '1' + - name: User-Agent + in: header + schema: + type: string + example: >- + Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like + Gecko) Chrome/114.0.0.0 Safari/537.36 + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/search: post: tags: - - Query Templates - summary: query template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + - Alerts Wrapper + summary: 'search rule ' requestBody: content: application/json: schema: type: object example: - id: api.query.template.query - ver: v2 - ts: '2024-04-10T16:10:50+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - request: - startdate: '2020-12-31' - enddate: '2024-12-31' - aggregationLevel: month - dataset: test - limit: 5 + request: {} responses: '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: JSON template with request body' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-14T11:52:24+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d - responseCode: OK - result: - - timestamp: '2023-09-01T00:00:00.000Z' - result: - school_id: 0 - example-1: - summary: 'Success: SQL template query with request body' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-14T11:33:06+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 - responseCode: OK - result: - - __time: '2023-09-11T00:00:00.000Z' - school_category: secondary - gender: others - state_id: '15' - district_id: '2002' - block_id: '70' - cluster_id: '485' - obsrv.meta.source.connector: null - obsrv.meta.source.id: null - grade_sum: 18 - school_id_sum: 180378 - students_marked_sum: 12492 - students_present_sum: 2466 - total_count: 18 - total_students_sum: 12492 - '400': - description: Bad Request + description: Successful response content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: invalid date range (native template)' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-02T12:59:14+05:30' - params: - status: FAILED - resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' - example-1: - summary: 'Failure: invalid date range' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:28:18+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' - '404': - description: Not Found + application/json: {} + /alerts/v1/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'get specific rule ' + responses: + '200': + description: Successful response content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:21:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b35a7050-b94c-4944-9630-233c9542272e - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: >- - Dataset test with table hour is not available for - querying - trace: '' - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:23:47+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: Datasource test11 not available for querying - trace: '' - /v2/dataset/copy: + application/json: {} + /alerts/v1/metric/alias/create: post: tags: - - Dataset copy - summary: Dataset copy + - Alert Metric_alias + summary: add metric requestBody: content: application/json: schema: type: object example: - id: api.datasets.copy - ver: '1.0' - ts: '2024-05-21T14:30:00Z' - params: - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - request: - source: - datasetId: test_dataset - isLive: true - destination: - datasetId: test-dataset-copy-1 + alias: Druid + component: MyTest + metric: sum(druid_supervisors{state='SUSPENDED'}) responses: '200': - description: OK + description: Successful response content: - application/json: - schema: - type: object - example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:57:11+05:30' - params: - status: SUCCESS - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: 77080c18-67b3-489c-af7b-9739709e3c4b - responseCode: OK - result: - dataset_id: sample1_copy.1 - message: Dataset clone successful - '400': - description: Bad Request + application/json: {} + /alerts/v1/metric/alias/search: + post: + tags: + - Alert Metric_alias + summary: list metrics + requestBody: + content: + application/json: + schema: + type: object + example: {} + responses: + '200': + description: Successful response content: - application/json: - schema: - type: object + application/json: {} + /alerts/v1/metric/alias/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Metric_alias + summary: update metric + requestBody: + content: + application/json: + schema: + type: object example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:56:26+05:30' - params: - status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_EXISTS - message: Dataset with id sample1 already exists - trace: '' - '404': - description: Not Found + alias: Druid native + responses: + '200': + description: Successful response content: - application/json: - schema: - type: object - example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:59:59+05:30' - params: - status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: NOT_FOUND - error: - code: DATASET_NOT_EXISTS - message: Dataset sample does not exists - trace: '' + application/json: {} + /alerts/v1/metric/alias/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Metric_alias + summary: remove metric + responses: + '200': + description: Successful response + content: + application/json: {} \ No newline at end of file From 4bdd81de3041d7cd2d4e9f484dd579152924f17b Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 19 Nov 2024 15:28:34 +0530 Subject: [PATCH 221/311] #OBS-I352 : updated swagger doc --- api-service/swagger-doc/openapi_v2.yml | 1461 ++---------------------- 1 file changed, 97 insertions(+), 1364 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 21dc9e3e..754f2a6f 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -100,35 +100,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '287' - ETag: - schema: - type: string - example: W/"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ" - Date: - schema: - type: string - example: Mon, 15 Jul 2024 13:14:09 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -192,35 +163,6 @@ paths: version_key: '1721220593027' '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '362' - ETag: - schema: - type: string - example: W/"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 03:09:00 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -242,35 +184,6 @@ paths: 'dataset_id' '409': description: Conflict - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '337' - ETag: - schema: - type: string - example: W/"151-a7dJ9XBUyT3AXNxl1TPcraxMX08" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 03:07:28 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -337,35 +250,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '1344' - ETag: - schema: - type: string - example: W/"540-790rZel+H/rDwgvZRxvlUmZ8Gpc" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 02:56:19 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -416,35 +300,6 @@ paths: https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '355' - ETag: - schema: - type: string - example: W/"163-9oQYJJEaBH3mJAnzDHXn2MxE848" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 03:03:04 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -597,35 +452,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '331' - ETag: - schema: - type: string - example: W/"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 13:00:45 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -663,35 +489,6 @@ paths: version_key: '1721134675878' '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '369' - ETag: - schema: - type: string - example: W/"171-iNJoyWUecOEsXbHZwx6rld3Sr1I" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 12:59:21 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -733,35 +530,6 @@ paths: to update the dataset '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '340' - ETag: - schema: - type: string - example: W/"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 12:58:30 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -781,35 +549,6 @@ paths: message: Dataset does not exists with id:telemetry_record-t41 '409': description: Conflict - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '383' - ETag: - schema: - type: string - example: W/"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw" - Date: - schema: - type: string - example: Tue, 16 Jul 2024 12:53:16 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -863,35 +602,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '501' - ETag: - schema: - type: string - example: W/"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg" - Date: - schema: - type: string - example: Wed, 17 Jul 2024 12:08:55 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -1012,35 +722,6 @@ paths: redis_db: 0 '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '307' - ETag: - schema: - type: string - example: W/"133-TQ9WpmutsrDcTNkRRmbWOhUChMk" - Date: - schema: - type: string - example: Wed, 17 Jul 2024 12:20:17 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -1061,35 +742,6 @@ paths: found. '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '304' - ETag: - schema: - type: string - example: W/"130-JL/oBB+GUHTrBp278giBHRvO71I" - Date: - schema: - type: string - example: Wed, 17 Jul 2024 12:21:12 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -1134,35 +786,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '24340' - ETag: - schema: - type: string - example: W/"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4" - Date: - schema: - type: string - example: Wed, 17 Jul 2024 12:25:36 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -3666,35 +3289,6 @@ paths: count: 2 '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '312' - ETag: - schema: - type: string - example: W/"138-XQplwhrgIYKIg0qtQdRCYWIGTNM" - Date: - schema: - type: string - example: Wed, 17 Jul 2024 12:32:26 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -4005,35 +3599,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '10760' - ETag: - schema: - type: string - example: W/"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I" - Date: - schema: - type: string - example: Mon, 22 Jul 2024 07:02:50 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -4547,35 +4112,6 @@ paths: datasource: boolean '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '364' - ETag: - schema: - type: string - example: W/"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0" - Date: - schema: - type: string - example: Mon, 22 Jul 2024 07:03:47 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -5567,35 +5103,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '346' - ETag: - schema: - type: string - example: W/"15a-SX5o6plRdfy+akEYhbMwq9zl+oU" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:34:26 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -5658,35 +5165,6 @@ paths: version_key: '1732001732045' '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '364' - ETag: - schema: - type: string - example: W/"16c-Dxem7prJM89DOqth+bdZbh0UI1g" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:38:05 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -5708,35 +5186,6 @@ paths: 'id' '409': description: Conflict - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '351' - ETag: - schema: - type: string - example: W/"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:31:11 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -5777,35 +5226,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '5155' - ETag: - schema: - type: string - example: W/"1423-Tx46QCrIX8So3vSGVkd865IYeEY" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:16:01 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -6356,35 +5776,6 @@ paths: updated_date: '2024-11-12T08:32:27.931Z' '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '291' - ETag: - schema: - type: string - example: W/"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:20:43 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -6403,35 +5794,6 @@ paths: message: Dataset with status:Draft cannot be exported '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '301' - ETag: - schema: - type: string - example: W/"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:15:10 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -6481,35 +5843,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '296' - ETag: - schema: - type: string - example: W/"128-bi4wSfgkyMP13qFE8VPTmgDdgLA" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:26:12 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -6528,35 +5861,6 @@ paths: message: Dataset clone successful '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '328' - ETag: - schema: - type: string - example: W/"148-jUDshghYf/jOhmmVkvtHlfJT864" - Date: - schema: - type: string - example: Tue, 19 Nov 2024 07:22:41 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -6597,35 +5901,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '250' - ETag: - schema: - type: string - example: W/"fa-+eWKIfUxsWBGuJy23qSucgLXke4" - Date: - schema: - type: string - example: Tue, 30 Jul 2024 09:55:51 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7067,35 +6342,6 @@ paths: count: 7 '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '399' - ETag: - schema: - type: string - example: W/"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k" - Date: - schema: - type: string - example: Tue, 30 Jul 2024 09:43:14 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7130,35 +6376,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '3304' - ETag: - schema: - type: string - example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" - Date: - schema: - type: string - example: Wed, 31 Jul 2024 12:47:54 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7366,35 +6583,6 @@ paths: live_date: '2024-06-25T04:38:28.847Z' '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '276' - ETag: - schema: - type: string - example: W/"114-izVC8DsHdeSfau/USVJvnIqZIMQ" - Date: - schema: - type: string - example: Thu, 01 Aug 2024 09:32:48 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7449,35 +6637,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '261' - ETag: - schema: - type: string - example: W/"105-UtaCmh0qZMBRBdniNq74Gr+4YAo" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:05:48 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7511,35 +6670,7 @@ paths: message: Data ingested successfully '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '300' - ETag: - schema: - type: string - example: W/"12c-GNCIUUDxOZD3UfM311sjnmFIgPc" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:03:32 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 + content: application/json: schema: @@ -7559,35 +6690,6 @@ paths: trace: '' '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '316' - ETag: - schema: - type: string - example: W/"13c-O9iirC/EyneYXQzth7iwEEy1UV4" - Date: - schema: - type: string - example: Thu, 18 Apr 2024 10:17:00 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7665,35 +6767,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '313' - ETag: - schema: - type: string - example: W/"139-De+IthAwrGNR+J11CwlNf5RSMmw" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:17:45 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7777,35 +6850,6 @@ paths: total_students_sum: 12492 '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '371' - ETag: - schema: - type: string - example: W/"173-OP8NcbSqLKFO92PIyUmMk0lNsXs" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:16:29 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7828,35 +6872,6 @@ paths: trace: '' '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '356' - ETag: - schema: - type: string - example: W/"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:13:12 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -7947,35 +6962,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '537' - ETag: - schema: - type: string - example: W/"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:24:14 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8025,35 +7011,6 @@ paths: updated_date: '2024-05-13T07:29:04.117Z' '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '294' - ETag: - schema: - type: string - example: W/"126-zq5j1yor+xr1XFEEbP09fQTDI/k" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:25:16 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8086,35 +7043,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '241' - ETag: - schema: - type: string - example: W/"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo" - Date: - schema: - type: string - example: Tue, 30 Apr 2024 06:28:27 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8131,35 +7059,6 @@ paths: message: Template yashash-k deleted successfully '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '303' - ETag: - schema: - type: string - example: W/"12f-99pWw8VTwuVfDAhinC55JXfNyyg" - Date: - schema: - type: string - example: Thu, 30 May 2024 18:28:41 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8209,35 +7108,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '359' - ETag: - schema: - type: string - example: W/"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4" - Date: - schema: - type: string - example: Mon, 29 Apr 2024 16:59:01 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8275,35 +7145,6 @@ paths: message: The query template has been saved successfully '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '407' - ETag: - schema: - type: string - example: W/"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8" - Date: - schema: - type: string - example: Mon, 29 Apr 2024 17:05:10 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8347,35 +7188,6 @@ paths: trace: '' '409': description: Conflict - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '350' - ETag: - schema: - type: string - example: W/"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ" - Date: - schema: - type: string - example: Mon, 29 Apr 2024 17:03:28 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -8419,35 +7231,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '6864' - ETag: - schema: - type: string - example: W/"1ad0-xp24UiXXXiFWplmv5Acja7prSYM" - Date: - schema: - type: string - example: Mon, 29 Apr 2024 13:46:03 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -9039,35 +7822,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '314' - ETag: - schema: - type: string - example: W/"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k" - Date: - schema: - type: string - example: Fri, 10 May 2024 05:51:47 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -9086,35 +7840,6 @@ paths: templateId: sql11template1 '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '429' - ETag: - schema: - type: string - example: W/"1ad-5sb8WUekFL8s4c1Ink6bUByoHho" - Date: - schema: - type: string - example: Fri, 10 May 2024 05:53:54 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -9224,35 +7949,6 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '301' - ETag: - schema: - type: string - example: W/"12d-9hKB38iHEwYPT2MgF8puXcq05Ew" - Date: - schema: - type: string - example: Tue, 14 May 2024 06:22:24 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -9302,35 +7998,6 @@ paths: total_students_sum: 12492 '400': description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '336' - ETag: - schema: - type: string - example: W/"150-T/XeSIt7PR7GcGEbET1e8n9zX7k" - Date: - schema: - type: string - example: Thu, 02 May 2024 07:29:14 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -9372,35 +8039,6 @@ paths: trace: '' '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '357' - ETag: - schema: - type: string - example: W/"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U" - Date: - schema: - type: string - example: Mon, 13 May 2024 07:51:46 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -10000,4 +8638,99 @@ paths: '200': description: Successful response content: - application/json: {} \ No newline at end of file + application/json: {} + /v2/connector/register: + post: + summary: Register a connector + description: Registers a connector by uploading a file. + operationId: registerConnector + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + file3: + type: string + format: binary + description: Connector distribution file to upload. + responses: + '200': + description: Connector registered successfully. + content: + application/json: + schema: + type: object + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:28:47+00:00" + params: + type: object + properties: + status: + type: string + example: SUCCESS + resmsgid: + type: string + example: fc2ce24f-a333-479f-931c-024d61039801 + responseCode: + type: string + example: OK + result: + type: object + properties: + message: + type: string + example: connector registered successfully. + '500': + description: Internal server error while registering the connector. + content: + application/json: + schema: + type: object + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:50:21+00:00" + params: + type: object + properties: + status: + type: string + example: FAILED + resmsgid: + type: string + example: ebaad9d7-c0d4-47e4-8cde-87fb25eef739 + responseCode: + type: string + example: INTERNAL_SERVER_ERROR + error: + type: object + properties: + code: + type: string + example: FAILED_TO_REGISTER_CONNECTOR + message: + type: string + example: Failed to Extract the File + trace: + type: string + example: "" + tags: + - Connector Register \ No newline at end of file From 46c7c86d302e7bd2846df282c270c0757c7955fb Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 19 Nov 2024 17:13:17 +0530 Subject: [PATCH 222/311] #000: Obsrv Installation Documentation - 1.2.0-RC --- api-service/swagger-doc/openapi_v2.yml | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 2e29d36a..1a064494 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -9443,7 +9443,7 @@ paths: post: tags: - Alert Notification-channels - summary: search channels + summary: Search Notificaation Channels requestBody: content: {} responses: @@ -9455,7 +9455,7 @@ paths: post: tags: - Alert Notification-channels - summary: Add channel (Email) + summary: Configure channel requestBody: content: application/json: @@ -9484,7 +9484,7 @@ paths: get: tags: - Alert Notification-channels - summary: publish channel + summary: Publish Channel responses: '200': description: Successful response @@ -9494,7 +9494,7 @@ paths: post: tags: - Alert Notification-channels - summary: test channel + summary: Verify Configured Channel requestBody: content: application/json: @@ -9527,7 +9527,7 @@ paths: patch: tags: - Alert Notification-channels - summary: update channel + summary: Update Specific Channel requestBody: content: application/json: @@ -9551,7 +9551,7 @@ paths: delete: tags: - Alert Notification-channels - summary: delete channel + summary: Delete Channel responses: '200': description: Successful response @@ -9632,7 +9632,7 @@ paths: patch: tags: - Alert silence - summary: Edit Request + summary: Modify Silence Request requestBody: content: application/json: @@ -9657,7 +9657,7 @@ paths: delete: tags: - Alerts Wrapper - summary: 'delete rule ' + summary: 'Delete Configured Alert Rules ' description: This URLwill provided access to user to delete any custom rule. responses: '200': @@ -9675,7 +9675,7 @@ paths: patch: tags: - Alerts Wrapper - summary: 'edit rule ' + summary: 'Update Rule ' description: >- This URL will provide access to users to edit any properties in the Alert rule and save the changes. @@ -9707,7 +9707,7 @@ paths: get: tags: - Alerts Wrapper - summary: 'publish rule ' + summary: 'Publish Alert Rules ' responses: '200': description: Successful response @@ -9896,7 +9896,7 @@ paths: post: tags: - Alerts Wrapper - summary: 'search rule ' + summary: 'Search Alert Rule' requestBody: content: application/json: @@ -9920,7 +9920,7 @@ paths: get: tags: - Alerts Wrapper - summary: 'get specific rule ' + summary: 'Get Alert Rule' responses: '200': description: Successful response @@ -9930,7 +9930,7 @@ paths: post: tags: - Alert Metric_alias - summary: add metric + summary: Create Custom Metric requestBody: content: application/json: @@ -9949,7 +9949,7 @@ paths: post: tags: - Alert Metric_alias - summary: list metrics + summary: List Create Metrics requestBody: content: application/json: @@ -9972,7 +9972,7 @@ paths: patch: tags: - Alert Metric_alias - summary: update metric + summary: Modify Metric requestBody: content: application/json: @@ -9996,7 +9996,7 @@ paths: delete: tags: - Alert Metric_alias - summary: remove metric + summary: Delete Metric responses: '200': description: Successful response From ec653b26eed31af59dd7e8c838d66444ccd292c0 Mon Sep 17 00:00:00 2001 From: Jerald <127138957+JeraldJF@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:15:19 +0530 Subject: [PATCH 223/311] #OBS-I352: feat: Postman collection of open source apis (#282) --- .../Obsrv v2 apis.postman_collection.json | 3099 ++++++++++++----- 1 file changed, 2229 insertions(+), 870 deletions(-) diff --git a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json index b48c505b..041ee138 100644 --- a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -5,7 +5,7 @@ "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", "_exporter_id": "26192103", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" + "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" }, "item": [ { @@ -23,7 +23,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" @@ -1215,7 +1215,7 @@ } ], "url": { - "raw": "localhost:3000/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "raw": "localhost:3000/v2/datasets/read/master-test?fields=name,type,id", "host": [ "localhost" ], @@ -1227,10 +1227,6 @@ "master-test" ], "query": [ - { - "key": "status", - "value": "Draft" - }, { "key": "fields", "value": "name,type,id" @@ -1649,7 +1645,7 @@ "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" }, { - "name": "Success: Filter basen on status as string", + "name": "Success: Filter based on status as string", "originalRequest": { "method": "POST", "header": [], @@ -2057,7 +2053,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset deleted successfully\",\n \"dataset_id\": \"master.1\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Delete successful\",\n \"dataset_id\": \"trip-data\"\n }\n}" }, { "name": "Delete failure: Dataset not found to delete", @@ -2080,7 +2076,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to delete\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: master\",\n \"trace\": \"\"\n }\n}" }, { "name": "Live success: Dataset published successfully", @@ -2103,7 +2099,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset published successfully\",\n \"dataset_id\": \"telemetry\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Live successful\",\n \"dataset_id\": \"telemetry_record\"\n }\n}" }, { "name": "Live failure: Dataset not found to publish", @@ -2126,7 +2122,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to publish\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: telemetry-data\",\n \"trace\": \"\"\n }\n}" }, { "name": "Live failure: Dataset in draft state", @@ -2149,7 +2145,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_NOT_READY_FOR_PUBLISH\",\n \"message\": \"Failed to publish dataset as it is in draft state\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_LIVE_FAILURE\",\n \"message\": \"Transition failed for dataset: sb-telemetry2 status:Draft with status transition to Live\",\n \"trace\": \"\"\n }\n}" }, { "name": "ReadyToPublish success: Dataset is ready to publish", @@ -2218,7 +2214,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Failed to mark dataset Ready to publish as it not in draft state\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Transition failed for dataset: telemetry-events status:Retired with status transition to ReadyToPublish\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire success: Dataset retired successfully", @@ -2241,7 +2237,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset retired successfully\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Retire successful\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" }, { "name": "Retire Failure: Dataset not found to retire", @@ -2264,7 +2260,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to retire\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: sb-telemetry2\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire Failure: Dataset is already retired", @@ -2287,7 +2283,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_ALREADY_RETIRED\",\n \"message\": \"Dataset is already retired\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_RETIRE_FAILURE\",\n \"message\": \"Transition failed for dataset: master-telemetrry status:Retired with status transition to Retire\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire failure: Cannot retire master dataset as it is used by other datasets", @@ -2310,7 +2306,7 @@ "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is used by other datasets\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset\",\n \"trace\": \"\"\n }\n}" }, { "name": "Failure: Invalid request payload provided", @@ -2334,29 +2330,6 @@ "header": [], "cookie": [], "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:03:56+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"ba4c86bd-b438-4582-b178-2410a5c5dd15\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_STATUS_INVALID_INPUT\",\n \"message\": \"#properties/request/properties/status/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to the db failed", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Delete\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/datasets/status-transition" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:24:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"92928434-719f-47d4-9946-1e40ecd53253\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"DATASET_STATUS_FAILURE\",\n \"message\": \"Failed to perform status transition on datasets\",\n \"trace\": \"\"\n }\n}" } ] }, @@ -2372,7 +2345,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" @@ -2380,7 +2353,7 @@ } }, "url": { - "raw": "localhost:3000/v2/datasets/import?override=true", + "raw": "localhost:3000/v2/datasets/import?overwrite=false", "host": [ "localhost" ], @@ -2392,105 +2365,53 @@ ], "query": [ { - "key": "override", - "value": "true" - } - ] - } - }, - "response": [] - }, - { - "name": "Dataset export", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "datasets", - "export", - "v1-copy" - ], - "query": [ - { - "key": "status", - "value": "Live" + "key": "overwrite", + "value": "false" } ] } }, - "response": [] - }, - { - "name": "Dataset copy", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/datasets/copy" - }, - "response": [] - } - ], - "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." - }, - { - "name": "Connector api's", - "item": [ - { - "name": "Connector list", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, "response": [ { - "name": "Failure: Invalid request body, filter option array should not be empty", + "name": "Success: Fully imported dataset", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2503,15 +2424,15 @@ }, { "key": "Content-Length", - "value": "399" + "value": "346" }, { "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + "value": "W/\"15a-SX5o6plRdfy+akEYhbMwq9zl+oU\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" + "value": "Tue, 19 Nov 2024 07:34:26 GMT" }, { "key": "Connection", @@ -2523,23 +2444,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:04:26+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"fd87393e-4b1b-4cf0-8e9a-a0529d691cc3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is imported successfully\",\n \"data\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"version_key\": \"1732001666671\"\n }\n }\n}" }, { - "name": "Success: Filtered based on status", + "name": "Success: Partially imported dataset", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } }, "status": "OK", "code": 200, @@ -2555,15 +2498,15 @@ }, { "key": "Content-Length", - "value": "250" + "value": "520" }, { "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + "value": "W/\"208-vju3WNZ7seMZsQ5AS29Wk0j+yOA\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" + "value": "Tue, 19 Nov 2024 07:29:41 GMT" }, { "key": "Connection", @@ -2575,23 +2518,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:59:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"379ced0b-eb06-445f-a7e3-8737cb16c351\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is partially imported\",\n \"data\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"version_key\": \"1732001381825\"\n },\n \"ignoredFields\": {\n \"denorm_fields\": [\n {\n \"config\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-dataset\"\n },\n \"details\": \"Master dataset does not exist\"\n }\n ]\n }\n }\n}" }, { - "name": "Success: Filtered based on category", + "name": "Success: Imported new dataset [overwrite=false]", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft\",\n \"dataset_id\": \"sb-telemetryRPF_Draft\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, "status": "OK", "code": 200, @@ -2607,15 +2572,15 @@ }, { "key": "Content-Length", - "value": "2571" + "value": "341" }, { "key": "ETag", - "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + "value": "W/\"155-UdyHMmAFuHt3xNP/3pFL5e1esfA\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:25:03 GMT" + "value": "Tue, 19 Nov 2024 07:35:32 GMT" }, { "key": "Connection", @@ -2627,26 +2592,48 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:05:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"1a50e1b1-14b0-4178-aba4-9e09a63d8d1d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is imported successfully\",\n \"data\": {\n \"id\": \"sb-telemetry_draft\",\n \"version_key\": \"1732001732045\"\n }\n }\n}" }, { - "name": "Success: Connectors list with all filter options", + "name": "Failure: Import Dataset with id already exists", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, - "status": "OK", - "code": 200, + "status": "Conflict", + "code": 409, "_postman_previewlanguage": "json", "header": [ { @@ -2659,15 +2646,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "351" }, { "key": "ETag", - "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + "value": "W/\"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:26:32 GMT" + "value": "Tue, 19 Nov 2024 07:31:11 GMT" }, { "key": "Connection", @@ -2679,26 +2666,48 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:01:11+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"1e5b34cd-4473-4439-8cc5-5ff917f47e48\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset with dataset_id: sb-telemetryRPF_draft_data already exists.\"\n }\n}" }, { - "name": "Success: Connectors list without filters", + "name": "Failure: Invalid configs to import", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -2711,15 +2720,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "364" }, { "key": "ETag", - "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" + "value": "W/\"16c-Dxem7prJM89DOqth+bdZbh0UI1g\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:27:37 GMT" + "value": "Tue, 19 Nov 2024 07:38:05 GMT" }, { "key": "Connection", @@ -2731,24 +2740,146 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - } - ] - }, - { - "name": "Connector Read", - "request": { + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:08:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"244318a5-d933-41a5-add6-14f299281069\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_IMPORT_INVALID_CONFIGS\",\n \"message\": \"#properties/request/required must have required property 'id'\"\n }\n}" + }, + { + "name": "Dataset import", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } + }, + "_postman_previewlanguage": null, + "header": null, + "cookie": [], + "body": null + }, + { + "name": "Dataset import", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } + }, + "_postman_previewlanguage": null, + "header": null, + "cookie": [], + "body": null + } + ] + }, + { + "name": "Dataset export", + "request": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } }, "response": [ { - "name": "Success: Read live Connector", + "name": "Success: Live Dataset export success", "originalRequest": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + "url": { + "raw": "localhost:3000/v2/datasets/export/content-data-v2?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "content-data-v2" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } }, "status": "OK", "code": 200, @@ -2764,15 +2895,15 @@ }, { "key": "Content-Length", - "value": "3304" + "value": "5155" }, { "key": "ETag", - "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + "value": "W/\"1423-Tx46QCrIX8So3vSGVkd865IYeEY\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 12:47:54 GMT" + "value": "Tue, 19 Nov 2024 07:16:01 GMT" }, { "key": "Connection", @@ -2784,30 +2915,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:46:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ae57c620-2858-40f9-9d8e-33f8f259449d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"content-data-v2\",\n \"dataset_id\": \"content-data-v2\",\n \"type\": \"master\",\n \"name\": \"content-data-v2\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"key\": \"tripID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"VendorID\": {\n \"key\": \"VendorID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tpep_pickup_datetime\": {\n \"key\": \"tpep_pickup_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"tpep_dropoff_datetime\": {\n \"key\": \"tpep_dropoff_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"passenger_count\": {\n \"key\": \"passenger_count\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"trip_distance\": {\n \"key\": \"trip_distance\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"RatecodeID\": {\n \"key\": \"RatecodeID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"store_and_fwd_flag\": {\n \"key\": \"store_and_fwd_flag\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"PULocationID\": {\n \"key\": \"PULocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"DOLocationID\": {\n \"key\": \"DOLocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"payment_type\": {\n \"key\": \"payment_type\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"primary_passenger\": {\n \"key\": \"primary_passenger\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"email\": {\n \"key\": \"email\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mobile\": {\n \"key\": \"mobile\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n }\n },\n \"fare_details\": {\n \"key\": \"fare_details\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"fare_amount\": {\n \"key\": \"fare_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"extra\": {\n \"key\": \"extra\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mta_tax\": {\n \"key\": \"mta_tax\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tip_amount\": {\n \"key\": \"tip_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tolls_amount\": {\n \"key\": \"tolls_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"improvement_surcharge\": {\n \"key\": \"improvement_surcharge\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"total_amount\": {\n \"key\": \"total_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"congestion_surcharge\": {\n \"key\": \"congestion_surcharge\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n }\n }\n },\n \"additionalProperties\": true\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"tripID\",\n \"dedup_period\": 604800\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"content-data-v2\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"container/api-service/user_uploads/chunk-2_addedf.json\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 7\n }\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"data_version\": 1,\n \"api_version\": \"v2\",\n \"version\": 1,\n \"sample_data\": {\n \"mergedEvent\": {\n \"tripID\": \"ccbb970c-4d10-4247-be0c-a480996f3429\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-26 00:36:10\",\n \"tpep_dropoff_datetime\": \"2023-09-15 00:45:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"263\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jacey_Hintz@yahoo.com\",\n \"mobile\": \"247-492-3370\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.25\",\n \"congestion_surcharge\": \"\"\n }\n }\n },\n \"entry_topic\": \"local.ingest\",\n \"created_date\": \"2024-10-14T06:23:44.588Z\",\n \"updated_date\": \"2024-10-14T06:23:44.588Z\"\n }\n}" }, { - "name": "Success: Read Draft Connector", + "name": "Success: ReadyToPublish Dataset exported successfully", "originalRequest": { "method": "GET", "header": [], "url": { - "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", - "protocol": "http", + "raw": "localhost:3000/v2/datasets/export/mydataset?status=Draft", "host": [ "localhost" ], "port": "3000", "path": [ "v2", - "connectors", - "read", - "mssql-connector-2.0.0" + "datasets", + "export", + "mydataset" ], "query": [ { - "key": "mode", - "value": "edit" + "key": "status", + "value": "Draft" } ] } @@ -2826,15 +2956,15 @@ }, { "key": "Content-Length", - "value": "744" + "value": "5635" }, { "key": "ETag", - "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + "value": "W/\"1603-tBOO40ZL+TVYU+yC9CWRvXGb7vs\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 07:17:12 GMT" + "value": "Tue, 19 Nov 2024 07:18:11 GMT" }, { "key": "Connection", @@ -2846,14 +2976,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:48:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"222847a0-32bf-4c63-be68-76f0e51258af\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mydataset\",\n \"dataset_id\": \"mydataset\",\n \"name\": \"mydataset\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"key\": \"tripID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"VendorID\": {\n \"key\": \"VendorID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tpep_pickup_datetime\": {\n \"key\": \"tpep_pickup_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"tpep_dropoff_datetime\": {\n \"key\": \"tpep_dropoff_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"passenger_count\": {\n \"key\": \"passenger_count\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"trip_distance\": {\n \"key\": \"trip_distance\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"RatecodeID\": {\n \"key\": \"RatecodeID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"store_and_fwd_flag\": {\n \"key\": \"store_and_fwd_flag\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"PULocationID\": {\n \"key\": \"PULocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"DOLocationID\": {\n \"key\": \"DOLocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"payment_type\": {\n \"key\": \"payment_type\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"primary_passenger\": {\n \"key\": \"primary_passenger\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"email\": {\n \"key\": \"email\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mobile\": {\n \"key\": \"mobile\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n },\n \"additionalProperties\": false\n },\n \"fare_details\": {\n \"key\": \"fare_details\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"fare_amount\": {\n \"key\": \"fare_amount\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"integer\"\n },\n \"1\": {\n \"type\": \"number\"\n }\n }\n },\n \"extra\": {\n \"key\": \"extra\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"mta_tax\": {\n \"key\": \"mta_tax\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tip_amount\": {\n \"key\": \"tip_amount\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"tolls_amount\": {\n \"key\": \"tolls_amount\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"integer\"\n },\n \"1\": {\n \"type\": \"number\"\n }\n }\n },\n \"improvement_surcharge\": {\n \"key\": \"improvement_surcharge\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"total_amount\": {\n \"key\": \"total_amount\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"congestion_surcharge\": {\n \"key\": \"congestion_surcharge\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true\n }\n },\n \"additionalProperties\": false\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"mydataset\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"container/api-service/user_uploads/chunk-1 (4) (1)_5ec855.json\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1731420131072\",\n \"api_version\": \"v2\",\n \"sample_data\": {\n \"mergedEvent\": {\n \"tripID\": \"4c77e9d5-538d-4eb7-8db1-4c2c32860aa8\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-11-02 00:18:42\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willa67@gmail.com\",\n \"mobile\": \"1-720-981-6399 x77055\"\n },\n \"fare_details\": {\n \"fare_amount\": 7,\n \"extra\": 0.5,\n \"mta_tax\": 0.5,\n \"tip_amount\": 0,\n \"tolls_amount\": 0,\n \"improvement_surcharge\": 0.3,\n \"total_amount\": 8.3,\n \"congestion_surcharge\": 0\n }\n }\n },\n \"entry_topic\": \"local.ingest\",\n \"created_date\": \"2024-11-11T21:31:21.390Z\",\n \"updated_date\": \"2024-11-12T08:32:27.931Z\"\n }\n}" }, { - "name": "Failure: Connector not found", + "name": "Failure: Dataset not found to export", "originalRequest": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } }, "status": "Not Found", "code": 404, @@ -2869,15 +3017,15 @@ }, { "key": "Content-Length", - "value": "276" + "value": "301" }, { "key": "ETag", - "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + "value": "W/\"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 09:32:48 GMT" + "value": "Tue, 19 Nov 2024 07:15:10 GMT" }, { "key": "Connection", @@ -2889,17 +3037,73 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:45:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"fd6fdae2-a002-4c15-81e2-3cce539a2a5a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:v1-copy not found to export\"\n }\n}" + }, + { + "name": "Failure: Invalid dataset state to export", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/new-dataset?status=Draft", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "new-dataset" + ], + "query": [ + { + "key": "status", + "value": "Draft" + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "291" + }, + { + "key": "ETag", + "value": "W/\"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY\"" + }, + { + "key": "Date", + "value": "Tue, 19 Nov 2024 07:20:43 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:50:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"2097228c-7872-4217-a06a-0ec08e3dc67d\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXPORT_FAILURE\",\n \"message\": \"Dataset with status:Draft cannot be exported\"\n }\n}" } ] - } - ] - }, - { - "name": "Data Ingest", - "item": [ + }, { - "name": "Data ingest", + "name": "Dataset copy", "request": { "method": "POST", "header": [ @@ -2910,40 +3114,39 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/sb-telemetry-imported" + "url": "localhost:3000/v2/datasets/copy" }, "response": [ { - "name": "Entry topic not found", + "name": "Success: Dataset cloned successfully", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", - "value": "application/json", - "type": "text" + "value": "application/json" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"content-data-v2\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"new-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/sample-test" + "url": "localhost:3000/v2/datasets/copy" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2956,15 +3159,15 @@ }, { "key": "Content-Length", - "value": "316" + "value": "296" }, { "key": "ETag", - "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" + "value": "W/\"128-bi4wSfgkyMP13qFE8VPTmgDdgLA\"" }, { "key": "Date", - "value": "Thu, 18 Apr 2024 10:17:00 GMT" + "value": "Tue, 19 Nov 2024 07:26:12 GMT" }, { "key": "Connection", @@ -2976,32 +3179,31 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:56:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"28072631-8e80-45eb-906f-d933a90646d0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"new-copy-live2\",\n \"message\": \"Dataset clone successful\"\n }\n}" }, { - "name": "Failure: schema validation", + "name": "Failure: Dataset not found to copy", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", - "value": "application/json", - "type": "text" + "value": "application/json" } ], "body": { "mode": "raw", - "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"content-data-v2\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"new-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test2" + "url": "localhost:3000/v2/datasets/copy" }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3014,15 +3216,15 @@ }, { "key": "Content-Length", - "value": "300" + "value": "328" }, { "key": "ETag", - "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" + "value": "W/\"148-jUDshghYf/jOhmmVkvtHlfJT864\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:03:32 GMT" + "value": "Tue, 19 Nov 2024 07:22:41 GMT" }, { "key": "Connection", @@ -3034,32 +3236,51 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Success: Data ingested successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "body": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:52:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"f2fd51bb-fc18-4278-a229-47fa25398e69\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset dataset-telemetry does not exists\"\n }\n}" + } + ] + } + ], + "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "response": [ + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test" + "url": "http://localhost:3000/v2/connectors/list" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -3072,15 +3293,15 @@ }, { "key": "Content-Length", - "value": "261" + "value": "399" }, { "key": "ETag", - "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:05:48 GMT" + "value": "Tue, 30 Jul 2024 09:43:14 GMT" }, { "key": "Connection", @@ -3092,29 +3313,23 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" }, { - "name": "Success: Data ingested successfully (batch)", + "name": "Success: Filtered based on status", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test2" + "url": "http://localhost:3000/v2/connectors/list" }, "status": "OK", "code": 200, @@ -3130,15 +3345,15 @@ }, { "key": "Content-Length", - "value": "261" + "value": "250" }, { "key": "ETag", - "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:06:49 GMT" + "value": "Tue, 30 Jul 2024 09:55:51 GMT" }, { "key": "Connection", @@ -3150,32 +3365,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" }, { - "name": "Failure: Dataset not found", + "name": "Success: Filtered based on category", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/added-tags" + "url": "http://localhost:3000/v2/connectors/list" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3188,15 +3397,15 @@ }, { "key": "Content-Length", - "value": "317" + "value": "2571" }, { "key": "ETag", - "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:07:29 GMT" + "value": "Wed, 31 Jul 2024 13:25:03 GMT" }, { "key": "Connection", @@ -3208,32 +3417,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" }, { - "name": "Failure: Entry topic not found", + "name": "Success: Connectors list with all filter options", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/sample1" + "url": "http://localhost:3000/v2/connectors/list" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3246,15 +3449,15 @@ }, { "key": "Content-Length", - "value": "316" + "value": "4380" }, { "key": "ETag", - "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:09:26 GMT" + "value": "Wed, 31 Jul 2024 13:26:32 GMT" }, { "key": "Connection", @@ -3266,60 +3469,23 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data query", - "item": [ - { - "name": "Data query", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" }, - "url": "localhost:3000/v2/data/query/test" - }, - "response": [ { - "name": "Success: native query (interval as array type)", + "name": "Success: Connectors list without filters", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": "http://localhost:3000/v2/connectors/list" }, "status": "OK", "code": 200, @@ -3335,15 +3501,15 @@ }, { "key": "Content-Length", - "value": "313" + "value": "4380" }, { "key": "ETag", - "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:17:45 GMT" + "value": "Wed, 31 Jul 2024 13:27:37 GMT" }, { "key": "Connection", @@ -3355,30 +3521,24 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" - }, + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "response": [ { - "name": "Success: native query (queryType: scan)", + "name": "Success: Read live Connector", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/data/query/test" + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" }, "status": "OK", "code": 200, @@ -3394,15 +3554,15 @@ }, { "key": "Content-Length", - "value": "313" + "value": "3304" }, { "key": "ETag", - "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:21:14 GMT" + "value": "Wed, 31 Jul 2024 12:47:54 GMT" }, { "key": "Connection", @@ -3414,54 +3574,57 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" }, { - "name": "Success: native query (normal intervals & queryType : timeseries)", + "name": "Success: Read Draft Connector", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/data/query/test" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "mssql-connector-2.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { "key": "Content-Length", - "value": "289" + "value": "744" }, { "key": "ETag", - "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:21:49 GMT" + "value": "Thu, 01 Aug 2024 07:17:12 GMT" }, { "key": "Connection", @@ -3473,30 +3636,101 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" }, { - "name": "Failure: Datasource not found in live table", + "name": "Failure: Connector not found", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "276" + }, + { + "key": "ETag", + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + }, + { + "key": "Date", + "value": "Thu, 01 Aug 2024 09:32:48 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Ingest", + "item": [ + { + "name": "Data ingest", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/in/sb-telemetry-imported" + }, + "response": [ + { + "name": "Entry topic not found", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/telemetry-eventssss" + "url": "localhost:3000/v2/data/in/sample-test" }, "status": "Not Found", "code": 404, @@ -3512,15 +3746,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "316" }, { "key": "ETag", - "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" + "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:13:12 GMT" + "value": "Thu, 18 Apr 2024 10:17:00 GMT" }, { "key": "Connection", @@ -3532,30 +3766,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Invalid date range", + "name": "Failure: schema validation", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/telemetry-events" + "url": "localhost:3000/v2/data/in/test2" }, "status": "Bad Request", "code": 400, @@ -3571,15 +3804,15 @@ }, { "key": "Content-Length", - "value": "371" + "value": "300" }, { "key": "ETag", - "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" + "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:16:29 GMT" + "value": "Thu, 30 May 2024 18:03:32 GMT" }, { "key": "Connection", @@ -3591,33 +3824,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Datasource not found in druid", + "name": "Success: Data ingested successfully", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": "localhost:3000/v2/data/in/test" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3630,15 +3862,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "261" }, { "key": "ETag", - "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" + "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:20:27 GMT" + "value": "Thu, 30 May 2024 18:05:48 GMT" }, { "key": "Connection", @@ -3650,30 +3882,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" }, { - "name": "Success: sql query", + "name": "Success: Data ingested successfully (batch)", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": "localhost:3000/v2/data/in/test2" }, "status": "OK", "code": 200, @@ -3689,15 +3920,15 @@ }, { "key": "Content-Length", - "value": "582" + "value": "261" }, { "key": "ETag", - "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" + "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:12:12 GMT" + "value": "Thu, 30 May 2024 18:06:49 GMT" }, { "key": "Connection", @@ -3709,30 +3940,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" }, { - "name": "SQL Failure: dataset not found in druid", + "name": "Failure: Dataset not found", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": "localhost:3000/v2/data/in/added-tags" }, "status": "Not Found", "code": 404, @@ -3748,15 +3978,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "317" }, { "key": "ETag", - "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" + "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:22:26 GMT" + "value": "Thu, 30 May 2024 18:07:29 GMT" }, { "key": "Connection", @@ -3768,30 +3998,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" }, { - "name": "SQL Failure: Datasource not found in live table", + "name": "Failure: Entry topic not found", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/undefined" + "url": "localhost:3000/v2/data/in/sample1" }, "status": "Not Found", "code": 404, @@ -3807,15 +4036,15 @@ }, { "key": "Content-Length", - "value": "346" + "value": "316" }, { "key": "ETag", - "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" + "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:23:02 GMT" + "value": "Thu, 30 May 2024 18:09:26 GMT" }, { "key": "Connection", @@ -3827,69 +4056,60 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" } ] } ] }, { - "name": "Data exhaust", + "name": "Data query", "item": [ { - "name": "Data exhaust", + "name": "Data query", "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "data", - "exhaust", - "beckn-test-data" - ], - "query": [ - { - "key": "from", - "value": "2024-02-01" - }, - { - "key": "to", - "value": "2024-02-22" - }, - { - "key": "type", - "value": "raw" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" } - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Query Templates", - "item": [ - { - "name": "Read template", - "request": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql_template_11" + } + }, + "url": "localhost:3000/v2/data/query/test" }, "response": [ { - "name": "Success: JSON template", + "name": "Success: native query (interval as array type)", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/jsontemplate111" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" }, "status": "OK", "code": 200, @@ -3905,15 +4125,15 @@ }, { "key": "Content-Length", - "value": "537" + "value": "313" }, { "key": "ETag", - "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" + "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:24:14 GMT" + "value": "Thu, 30 May 2024 18:17:45 GMT" }, { "key": "Connection", @@ -3925,14 +4145,30 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" }, { - "name": "Success: SQL template", + "name": "Success: native query (queryType: scan)", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql1" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" }, "status": "OK", "code": 200, @@ -3948,15 +4184,15 @@ }, { "key": "Content-Length", - "value": "528" + "value": "313" }, { "key": "ETag", - "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" + "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:24:38 GMT" + "value": "Thu, 30 May 2024 18:21:14 GMT" }, { "key": "Connection", @@ -3968,17 +4204,33 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" }, { - "name": "Failure : Template not exists", + "name": "Success: native query (normal intervals & queryType : timeseries)", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql100" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/test" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3991,15 +4243,15 @@ }, { "key": "Content-Length", - "value": "294" + "value": "289" }, { "key": "ETag", - "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:25:16 GMT" + "value": "Thu, 30 May 2024 18:21:49 GMT" }, { "key": "Connection", @@ -4011,27 +4263,33 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Delete template", - "request": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" - }, - "response": [ + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + }, { - "name": "Success: Deleted successfully", + "name": "Failure: Datasource not found in live table", "originalRequest": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/telemetry-eventssss" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4044,15 +4302,15 @@ }, { "key": "Content-Length", - "value": "241" + "value": "356" }, { "key": "ETag", - "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" + "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:28:27 GMT" + "value": "Thu, 30 May 2024 18:13:12 GMT" }, { "key": "Connection", @@ -4064,17 +4322,33 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Template does not exists", + "name": "Failure: Invalid date range", "originalRequest": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/json_template" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/data/query/telemetry-events" }, - "status": "Not Found", - "code": 404, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -4087,15 +4361,15 @@ }, { "key": "Content-Length", - "value": "303" + "value": "371" }, { "key": "ETag", - "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" + "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:28:41 GMT" + "value": "Thu, 30 May 2024 18:16:29 GMT" }, { "key": "Connection", @@ -4107,57 +4381,33 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Create query template", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" }, - "url": "localhost:3000/v2/template/create/json_template_111" - }, - "response": [ { - "name": "Success : Create query template success (json)", + "name": "Failure: Datasource not found in druid", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": "localhost:3000/v2/data/query/test" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4170,15 +4420,15 @@ }, { "key": "Content-Length", - "value": "359" + "value": "356" }, { "key": "ETag", - "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" + "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 16:59:01 GMT" + "value": "Thu, 30 May 2024 18:20:27 GMT" }, { "key": "Connection", @@ -4190,29 +4440,30 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success: Create query template success (SQL)", + "name": "Success: sql query", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": "localhost:3000/v2/data/query/test" }, "status": "OK", "code": 200, @@ -4228,15 +4479,15 @@ }, { "key": "Content-Length", - "value": "339" + "value": "582" }, { "key": "ETag", - "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" + "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 16:59:58 GMT" + "value": "Thu, 30 May 2024 18:12:12 GMT" }, { "key": "Connection", @@ -4248,32 +4499,33 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" }, { - "name": "Failure: Template already exists", + "name": "SQL Failure: dataset not found in druid", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": "localhost:3000/v2/data/query/test" }, - "status": "Conflict", - "code": 409, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4286,15 +4538,15 @@ }, { "key": "Content-Length", - "value": "350" + "value": "356" }, { "key": "ETag", - "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:03:28 GMT" + "value": "Thu, 30 May 2024 18:22:26 GMT" }, { "key": "Connection", @@ -4306,32 +4558,33 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Missing required variables", + "name": "SQL Failure: Datasource not found in live table", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": "localhost:3000/v2/data/query/undefined" }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4344,15 +4597,15 @@ }, { "key": "Content-Length", - "value": "407" + "value": "346" }, { "key": "ETag", - "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:05:10 GMT" + "value": "Thu, 30 May 2024 18:23:02 GMT" }, { "key": "Connection", @@ -4364,32 +4617,72 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" - }, + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data exhaust", + "item": [ + { + "name": "Data exhaust", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "exhaust", + "beckn-test-data" + ], + "query": [ + { + "key": "from", + "value": "2024-02-01" + }, + { + "key": "to", + "value": "2024-02-22" + }, + { + "key": "type", + "value": "raw" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Query Templates", + "item": [ + { + "name": "Read template", + "request": { + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/sql_template_11" + }, + "response": [ { - "name": "Failure: Schema validation failure", + "name": "Success: JSON template", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/create" + "method": "GET", + "header": [], + "url": "localhost:3000/v2/template/read/jsontemplate111" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4402,15 +4695,15 @@ }, { "key": "Content-Length", - "value": "416" + "value": "537" }, { "key": "ETag", - "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:13:57 GMT" + "value": "Thu, 30 May 2024 18:24:14 GMT" }, { "key": "Connection", @@ -4422,42 +4715,14 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "List templates", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" }, - "url": "localhost:3000/v2/template/list" - }, - "response": [ { - "name": "empty request body", + "name": "Success: SQL template", "originalRequest": { - "method": "POST", + "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": "localhost:3000/v2/template/read/sql1" }, "status": "OK", "code": 200, @@ -4473,15 +4738,15 @@ }, { "key": "Content-Length", - "value": "6864" + "value": "528" }, { "key": "ETag", - "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 13:46:03 GMT" + "value": "Thu, 30 May 2024 18:24:38 GMT" }, { "key": "Connection", @@ -4493,26 +4758,17 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" }, { - "name": "Success : order by", + "name": "Failure : Template not exists", "originalRequest": { - "method": "POST", + "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": "localhost:3000/v2/template/read/sql100" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4525,15 +4781,15 @@ }, { "key": "Content-Length", - "value": "7700" + "value": "294" }, { "key": "ETag", - "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:06:09 GMT" + "value": "Thu, 30 May 2024 18:25:16 GMT" }, { "key": "Connection", @@ -4545,23 +4801,24 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" - }, + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Delete template", + "request": { + "method": "DELETE", + "header": [], + "url": "localhost:3000/v2/template/delete/yashash-k" + }, + "response": [ { - "name": "Success : Filters", + "name": "Success: Deleted successfully", "originalRequest": { - "method": "POST", + "method": "DELETE", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": "localhost:3000/v2/template/delete/yashash-k" }, "status": "OK", "code": 200, @@ -4577,15 +4834,15 @@ }, { "key": "Content-Length", - "value": "2708" + "value": "241" }, { "key": "ETag", - "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:08:55 GMT" + "value": "Tue, 30 Apr 2024 06:28:27 GMT" }, { "key": "Connection", @@ -4597,26 +4854,17 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" }, { - "name": "Success : limit and offset", + "name": "Failure: Template does not exists", "originalRequest": { - "method": "POST", + "method": "DELETE", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": "localhost:3000/v2/template/delete/json_template" }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4629,15 +4877,15 @@ }, { "key": "Content-Length", - "value": "1921" + "value": "303" }, { "key": "ETag", - "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:11:36 GMT" + "value": "Thu, 30 May 2024 18:28:41 GMT" }, { "key": "Connection", @@ -4649,42 +4897,54 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" } ] }, { - "name": "update template", + "name": "Create query template", "request": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": "localhost:3000/v2/template/create" }, "response": [ { - "name": "Success: update successful", + "name": "Success : Create query template success (json)", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": "localhost:3000/v2/template/create" }, "status": "OK", "code": 200, @@ -4700,15 +4960,15 @@ }, { "key": "Content-Length", - "value": "314" + "value": "359" }, { "key": "ETag", - "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:51:47 GMT" + "value": "Mon, 29 Apr 2024 16:59:01 GMT" }, { "key": "Connection", @@ -4720,26 +4980,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" }, { - "name": "Failure: required variables not exists to update", + "name": "Success: Create query template success (SQL)", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": "localhost:3000/v2/template/create" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4752,15 +5018,15 @@ }, { "key": "Content-Length", - "value": "429" + "value": "339" }, { "key": "ETag", - "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:53:54 GMT" + "value": "Mon, 29 Apr 2024 16:59:58 GMT" }, { "key": "Connection", @@ -4772,26 +5038,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" }, { - "name": "Failure: Template name validation failure", + "name": "Failure: Template already exists", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": "localhost:3000/v2/template/create" }, - "status": "Bad Request", - "code": 400, + "status": "Conflict", + "code": 409, "_postman_previewlanguage": "json", "header": [ { @@ -4804,15 +5076,15 @@ }, { "key": "Content-Length", - "value": "405" + "value": "350" }, { "key": "ETag", - "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:56:59 GMT" + "value": "Mon, 29 Apr 2024 17:03:28 GMT" }, { "key": "Connection", @@ -4824,23 +5096,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: query_type should when updating query", + "name": "Failure: Missing required variables", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": "localhost:3000/v2/template/create" }, "status": "Bad Request", "code": 400, @@ -4856,15 +5134,15 @@ }, { "key": "Content-Length", - "value": "412" + "value": "407" }, { "key": "ETag", - "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" + "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 07:02:57 GMT" + "value": "Mon, 29 Apr 2024 17:05:10 GMT" }, { "key": "Connection", @@ -4876,23 +5154,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: query should present when updating query_type", + "name": "Failure: Schema validation failure", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": "localhost:3000/v2/template/create" }, "status": "Bad Request", "code": 400, @@ -4908,15 +5192,15 @@ }, { "key": "Content-Length", - "value": "412" + "value": "416" }, { "key": "ETag", - "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" + "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 07:04:55 GMT" + "value": "Mon, 29 Apr 2024 17:13:57 GMT" }, { "key": "Connection", @@ -4928,70 +5212,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" } ] }, { - "name": "query template", + "name": "List templates", "request": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": "localhost:3000/v2/template/list" }, "response": [ { - "name": "Failure: invalid date range (native template)", + "name": "empty request body", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", "options": { "raw": { "language": "json" } } }, - "url": { - "raw": "localhost:3000/v2/template/query/telemetry-events", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "template", - "query", - "telemetry-events" - ], - "query": [ - { - "key": "limit", - "value": "1", - "disabled": true - } - ] - } + "url": "localhost:3000/v2/template/list" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5004,15 +5263,15 @@ }, { "key": "Content-Length", - "value": "336" + "value": "6864" }, { "key": "ETag", - "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" }, { "key": "Date", - "value": "Thu, 02 May 2024 07:29:14 GMT" + "value": "Mon, 29 Apr 2024 13:46:03 GMT" }, { "key": "Connection", @@ -5024,26 +5283,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" }, { - "name": "Failure: Datasource not found in druid", + "name": "Success : order by", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": "localhost:3000/v2/template/list" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5056,15 +5315,15 @@ }, { "key": "Content-Length", - "value": "357" + "value": "7700" }, { "key": "ETag", - "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" + "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:51:46 GMT" + "value": "Tue, 30 Apr 2024 06:06:09 GMT" }, { "key": "Connection", @@ -5076,26 +5335,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" }, { - "name": "Failure: Datasource not found in live table", + "name": "Success : Filters", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": "localhost:3000/v2/template/list" }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5108,15 +5367,15 @@ }, { "key": "Content-Length", - "value": "343" + "value": "2708" }, { "key": "ETag", - "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" + "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:53:47 GMT" + "value": "Tue, 30 Apr 2024 06:08:55 GMT" }, { "key": "Connection", @@ -5128,26 +5387,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" }, { - "name": "Failure: invalid date range", + "name": "Success : limit and offset", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": "localhost:3000/v2/template/list" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5160,15 +5419,15 @@ }, { "key": "Content-Length", - "value": "371" + "value": "1921" }, { "key": "ETag", - "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:58:18 GMT" + "value": "Tue, 30 Apr 2024 06:11:36 GMT" }, { "key": "Connection", @@ -5180,29 +5439,42 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" - }, + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" + } + ] + }, + { + "name": "update template", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "response": [ { - "name": "Success: JSON template with request body", + "name": "Success: update successful", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "method": "PATCH", + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/jsontemplate1" + "url": "localhost:3000/v2/template/update/sql11template1" }, "status": "OK", "code": 200, @@ -5218,15 +5490,15 @@ }, { "key": "Content-Length", - "value": "301" + "value": "314" }, { "key": "ETag", - "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" + "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" }, { "key": "Date", - "value": "Tue, 14 May 2024 06:22:24 GMT" + "value": "Fri, 10 May 2024 05:51:47 GMT" }, { "key": "Connection", @@ -5238,26 +5510,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" }, { - "name": "Success: SQL template query with request body", + "name": "Failure: required variables not exists to update", "originalRequest": { - "method": "POST", + "method": "PATCH", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": "localhost:3000/v2/template/update/sql11template1" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -5270,15 +5542,15 @@ }, { "key": "Content-Length", - "value": "594" + "value": "429" }, { "key": "ETag", - "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" + "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" }, { "key": "Date", - "value": "Tue, 14 May 2024 06:03:06 GMT" + "value": "Fri, 10 May 2024 05:53:54 GMT" }, { "key": "Connection", @@ -5290,9 +5562,1096 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Template name validation failure", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "405" + }, + { + "key": "ETag", + "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:56:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query_type should when updating query", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "412" + }, + { + "key": "ETag", + "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 07:02:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query should present when updating query_type", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/update/sql11template1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "412" + }, + { + "key": "ETag", + "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 07:04:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" } ] + }, + { + "name": "query template", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "response": [ + { + "name": "Failure: invalid date range (native template)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "telemetry-events" + ], + "query": [ + { + "key": "limit", + "value": "1", + "disabled": true + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "336" + }, + { + "key": "ETag", + "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + }, + { + "key": "Date", + "value": "Thu, 02 May 2024 07:29:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "357" + }, + { + "key": "ETag", + "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:51:46 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "343" + }, + { + "key": "ETag", + "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:53:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: invalid date range", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:58:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: JSON template with request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/jsontemplate1" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "301" + }, + { + "key": "ETag", + "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" + }, + { + "key": "Date", + "value": "Tue, 14 May 2024 06:22:24 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + }, + { + "name": "Success: SQL template query with request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/template/query/sql_test_1" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "594" + }, + { + "key": "ETag", + "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" + }, + { + "key": "Date", + "value": "Tue, 14 May 2024 06:03:06 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + } + ] + } + ] + }, + { + "name": "Schema validator", + "item": [ + { + "name": "Schema validator", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"test_dataset\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/data/v2/schema/validate" + }, + "response": [ + { + "name": "Succes: Schema validation successful", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"sb-telemetry\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/schema/validate" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "266" + }, + { + "key": "ETag", + "value": "W/\"10a-hzmB3lhC1mzaRhnGxclXsZ1lfHY\"" + }, + { + "key": "Date", + "value": "Thu, 20 Jun 2024 11:19:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-20T16:49:21+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"af336811-cfd8-4436-9682-05e0f25e2575\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Success\",\n \"isValid\": true\n }\n}" + }, + { + "name": "Failure: Dataset not exists", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"test_dataset\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3000/v2/schema/validate" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "324" + }, + { + "key": "ETag", + "value": "W/\"144-Eg0khgCz78o32V2ogAkVSj5KgLw\"" + }, + { + "key": "Date", + "value": "Tue, 18 Jun 2024 11:39:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T17:09:38+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5ad8657b-b556-4563-9d39-e1170f1062e8\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset test_dataset does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Alert Notification-channels", + "item": [ + { + "name": "search channels", + "request": { + "method": "POST", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/notifications/search" + }, + "response": [] + }, + { + "name": "Add channel (Slack)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"manager\": \"grafana\",\n \"name\": \"functional-metrics-slack\",\n \"type\": \"slack\",\n \"config\": {\n \"channel\": \"grafana-alerts\",\n \"webhookUrl\": \"https://hooks.slack.com/services/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\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/notifications/create" + }, + "response": [] + }, + { + "name": "publish channel", + "request": { + "method": "GET", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/notifications/publish/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + }, + "response": [] + }, + { + "name": "test channel", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"message\": \"Testing Email integration. If you can read this, it's working!\",\n \"payload\": {\n \"error\": {\n \"selectChannel\": true,\n \"configureChannel\": true\n },\n \"manager\": \"grafana\",\n \"name\": \"udhw\",\n \"type\": \"email\",\n \"config\": {\n \"recipientAddresses\": \"jerald@sanketika.in\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/notifications/test" + }, + "response": [] + }, + { + "name": "update channel", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"updated name\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/notifications/update/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + }, + "response": [] + }, + { + "name": "delete channel", + "request": { + "method": "DELETE", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/notifications/delete/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + }, + "response": [] + }, + { + "name": "Add channel (Email)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"manager\": \"grafana\",\n \"name\": \"functional-metrics-email\",\n \"type\": \"email\",\n \"config\": {\n \"recipientAddresses\": \"yravinderkumar33@gmail.com;ravinder@sanketika.in\",\n \"subject\": \"Obsrv Prod Alert\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/notifications/create" + }, + "response": [] + } + ] + }, + { + "name": "Alert silence", + "item": [ + { + "name": "Add Silence", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"startDate\": \"2024-01-11T07:56:23Z\",\n \"endDate\": \"2024-01-11T08:56:23Z\",\n \"alertId\": \"c7464d32-1d8d-4eaf-9b23-1313a3ff8149\",\n \"manager\": \"grafana\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/silence/create" + }, + "response": [] + }, + { + "name": "Get Silence", + "request": { + "method": "GET", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/silence/get/0ec52a4c-4529-4f70-b6f1-0de4b1e81c13" + }, + "response": [] + }, + { + "name": "Delete Silence", + "request": { + "method": "DELETE", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/silence/delete/39c7eec1-db78-4f5d-8e77-38385cc3b7dc" + }, + "response": [] + }, + { + "name": "Fetch All Silences", + "request": { + "method": "GET", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/silence/search" + }, + "response": [] + }, + { + "name": "Edit Request", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"startDate\": \"2023-08-09T10:50:59Z\",\n \"endDate\": \"2023-08-10T10:30:59Z\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/silence/update/da46091b-65db-4df9-85d1-c2341168bcb8" + }, + "response": [] + } + ] + }, + { + "name": "Alerts Wrapper", + "item": [ + { + "name": "delete rule ", + "request": { + "method": "DELETE", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/delete/0c3e08d4-8e4f-4810-b591-f5a294ee378e", + "description": "This URLwill provided access to user to delete any custom rule." + }, + "response": [] + }, + { + "name": "edit rule ", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"description\":\"This alert rule is designed to promptly notify you when one or more servers in your infrastructure become unresponsive or inaccessible.\",\n \"labels\":{\n \"severity\":\"warning\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/update/a6c46c48-79c1-4e34-9552-fe8f13d3c73e", + "description": "This URL will provide access to users to edit any properties in the Alert rule and save the changes." + }, + "response": [] + }, + { + "name": "publish rule ", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "" + }, + "url": "{{HOST_IP}}/alerts/v1/publish/c6c93917-9e5e-4aa6-9986-1c53c45ba7fe" + }, + "response": [] + }, + { + "name": "Add Rule ", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*" + }, + { + "key": "Accept-Language", + "value": "en-GB,en" + }, + { + "key": "Cache-Control", + "value": "no-store" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; grafana_session=4eb514f6fef4ad6884e47e50254af650; grafana_session_expiry=1687239735" + }, + { + "key": "Origin", + "value": "http://localhost:3001" + }, + { + "key": "Pragma", + "value": "no-store" + }, + { + "key": "Referer", + "value": "http://localhost:3001/alertRules/add" + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty" + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors" + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin" + }, + { + "key": "Sec-GPC", + "value": "1" + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"Total Api Calls1\",\n \"manager\":\"grafana\",\n \"description\": \"This alert is set up to notify you when the CPU usage on a host reaches a Low level.\",\n \"expression\": \"(node_total_api_calls) > 20\",\n \"category\": \"Infra\",\n \"severity\":\"warning\",\n \"frequency\": \"1m\",\n \"interval\": \"1m\",\n \"labels\": {\n \"component\": \"api\",\n \"notificationChannel\": \"slack\"\n },\n \"annotations\":{\n \"summary\":\"Host Low CPU usage\"\n },\n \"metadata\": {\n \"query\": [\n {\n \"refId\": \"A\",\n \"datasourceUid\": \"$datasourceUid\",\n \"queryType\": \"\",\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n },\n \"model\": {\n \"refId\": \"A\",\n \"hide\": false,\n \"editorMode\": \"code\",\n \"expr\": \"node_total_api_calls\",\n \"legendFormat\": \"__auto\",\n \"range\": true\n }\n },\n {\n \"refId\": \"B\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"B\",\n \"hide\": false,\n \"type\": \"reduce\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"B\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"reducer\": \"last\",\n \"expression\": \"A\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n },\n {\n \"refId\": \"C\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"C\",\n \"hide\": false,\n \"type\": \"threshold\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [\n 20\n ],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"C\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"expression\": \"B\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n }\n ]\n }\n}" + }, + "url": "{{HOST_IP}}/alerts/v1/create" + }, + "response": [] + }, + { + "name": "search rule ", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/search" + }, + "response": [] + }, + { + "name": "get specific rule ", + "request": { + "method": "GET", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/get/dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff" + }, + "response": [] + } + ] + }, + { + "name": "Alert Metric_alias", + "item": [ + { + "name": "add metric", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"alias\": \"Druid\",\n \"component\": \"MyTest\",\n \"metric\": \"sum(druid_supervisors{state='SUSPENDED'})\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/metric/alias/create" + }, + "response": [] + }, + { + "name": "list metrics", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/metric/alias/search" + }, + "response": [] + }, + { + "name": "update metric", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"alias\":\"Druid native\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/alerts/v1/metric/alias/update/48aad878-d59c-4046-aeb5-fb87aefbcfed" + }, + "response": [] + }, + { + "name": "remove metric", + "request": { + "method": "DELETE", + "header": [], + "url": "{{HOST_IP}}/alerts/v1/metric/alias/delete/f168707f-5908-4e2e-9111-e73c41e2700d" + }, + "response": [] + } + ] + }, + { + "name": "Connector registry", + "item": [ + { + "name": "Connector registry upload", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream", + "type": "text", + "disabled": true + }, + { + "key": "Content-Type", + "value": "multipart/form-data", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file2", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/object_store_connector-0.1.0.tar", + "disabled": true + }, + { + "key": "file", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/zipfile.tar.gz", + "disabled": true + }, + { + "key": "file3", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/kafka-connector-1.0.0-distribution.tar.gz" + }, + { + "key": "file4", + "value": "hello", + "type": "text", + "disabled": true + } + ] + }, + "url": "localhost:3000/v2/connector/register" + }, + "response": [] } ] } From 505c3c699333cad3fe69d391518c465cdb63e392 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Tue, 19 Nov 2024 17:33:14 +0530 Subject: [PATCH 224/311] #000: Obsrv Installation Documentation - 1.2.0-RC --- api-service/swagger-doc/openapi_v2.yml | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 1a064494..70c3024c 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -28,7 +28,7 @@ paths: post: tags: - Dataset api's - summary: Dataset create + summary: Dataset Create description: >- This API allows you to create new datasets used by the analytical data source. @@ -310,7 +310,7 @@ paths: post: tags: - Dataset api's - summary: File generate url + summary: Generate Presigned URLs description: This API generates presigned URLs to upload or download files from cloud requestBody: content: @@ -486,7 +486,7 @@ paths: patch: tags: - Dataset api's - summary: Dataset update + summary: Update Dataset description: >- This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or @@ -841,7 +841,7 @@ paths: get: tags: - Dataset api's - summary: Read dataset + summary: Read Dataset description: >- This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through @@ -1113,7 +1113,7 @@ paths: post: tags: - Dataset api's - summary: Dataset list + summary: List Datasets description: >- This API allows you to list all datasets. User can apply filters on dataset status and type. @@ -3718,7 +3718,7 @@ paths: post: tags: - Dataset api's - summary: Data schema generator + summary: Schema Generation description: This api is used to generate data schema for the given dataset event. requestBody: content: @@ -5768,7 +5768,7 @@ paths: get: tags: - Dataset api's - summary: Dataset export + summary: Export Dataset parameters: - name: status in: query @@ -6455,7 +6455,7 @@ paths: post: tags: - Dataset api's - summary: Dataset copy + summary: Clone Dataset requestBody: content: application/json: @@ -6579,7 +6579,7 @@ paths: post: tags: - Connector api's - summary: Connector list + summary: List Connectors requestBody: content: application/json: @@ -7127,7 +7127,7 @@ paths: get: tags: - Connector api's - summary: Connector Read + summary: Read Connector responses: '200': description: OK @@ -7423,7 +7423,7 @@ paths: post: tags: - Data Ingest - summary: Data ingest + summary: Ingest Data requestBody: content: application/json: @@ -7646,7 +7646,7 @@ paths: post: tags: - Data query - summary: Data query + summary: Query Data requestBody: content: application/json: @@ -7944,7 +7944,7 @@ paths: get: tags: - Query Templates - summary: Read template + summary: Read Template responses: '200': description: OK @@ -8083,7 +8083,7 @@ paths: delete: tags: - Query Templates - summary: Delete template + summary: Delete Template responses: '200': description: OK @@ -8399,7 +8399,7 @@ paths: post: tags: - Query Templates - summary: List templates + summary: List Templates requestBody: content: application/json: @@ -9019,7 +9019,7 @@ paths: patch: tags: - Query Templates - summary: update template + summary: Update Template requestBody: content: application/json: @@ -9204,7 +9204,7 @@ paths: post: tags: - Query Templates - summary: query template + summary: Query Template requestBody: content: application/json: @@ -9976,7 +9976,7 @@ paths: requestBody: content: application/json: - schema: + schem: type: object example: alias: Druid native From d5405f969eee1542b476a68c6837d447c7d8b263 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Thu, 21 Nov 2024 10:43:08 +0530 Subject: [PATCH 225/311] #000: Obsrv Installation Documentation - 1.2.0-RC --- api-service/swagger-doc/openapi_v2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 70c3024c..1f0c570f 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -9976,7 +9976,7 @@ paths: requestBody: content: application/json: - schem: + schema: type: object example: alias: Druid native From afff73b3b4612beb55daf0b84e087f2b90feef46 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 21 Nov 2024 11:05:13 +0530 Subject: [PATCH 226/311] #OBS-I352 : added connector register api --- api-service/swagger-doc/openapi_v2.yml | 111 +++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 54f824b1..759c0997 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -17,10 +17,10 @@ tags: capability to list multiple records based on specific criteria. - name: Connector API's - name: Data Ingest - - name: Data query + - name: Data Query - name: Query Templates - name: Alert Notification-channels - - name: Alert silence + - name: Alert Silence - name: Alerts Wrapper - name: Alert Metric_alias paths: @@ -7411,6 +7411,101 @@ paths: error: code: CONNECTOR_NOT_FOUND message: 'Connector not found: postgres-conn' + /v2/connector/register: + post: + tags: + - Connector API's + summary: Register a connector + description: Registers a connector by uploading a file. + operationId: registerConnector + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + file3: + type: string + format: binary + description: Connector distribution file to upload. + responses: + '200': + description: Connector registered successfully. + content: + application/json: + schema: + type: object + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:28:47+00:00" + params: + type: object + properties: + status: + type: string + example: SUCCESS + resmsgid: + type: string + example: fc2ce24f-a333-479f-931c-024d61039801 + responseCode: + type: string + example: OK + result: + type: object + properties: + message: + type: string + example: connector registered successfully. + '500': + description: Internal server error while registering the connector. + content: + application/json: + schema: + type: object + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:50:21+00:00" + params: + type: object + properties: + status: + type: string + example: FAILED + resmsgid: + type: string + example: ebaad9d7-c0d4-47e4-8cde-87fb25eef739 + responseCode: + type: string + example: INTERNAL_SERVER_ERROR + error: + type: object + properties: + code: + type: string + example: FAILED_TO_REGISTER_CONNECTOR + message: + type: string + example: Failed to Extract the File + trace: + type: string + example: "" /v2/data/in/{dataset_id}: parameters: - name: dataset_id @@ -7644,7 +7739,7 @@ paths: /v2/data/query/test: post: tags: - - Data query + - Data Query summary: Query Data requestBody: content: @@ -9559,7 +9654,7 @@ paths: /alerts/v1/silence/create: post: tags: - - Alert silence + - Alert Silence summary: Add Silence requestBody: content: @@ -9586,7 +9681,7 @@ paths: description: Unique identifier for the alert get: tags: - - Alert silence + - Alert Silence summary: Get Silence responses: '200': @@ -9603,7 +9698,7 @@ paths: description: Unique identifier for the alert delete: tags: - - Alert silence + - Alert Silence summary: Delete Silence responses: '200': @@ -9613,7 +9708,7 @@ paths: /alerts/v1/silence/search: get: tags: - - Alert silence + - Alert Silence summary: Fetch All Silences responses: '200': @@ -9630,7 +9725,7 @@ paths: description: Unique identifier for the alert patch: tags: - - Alert silence + - Alert Silence summary: Modify Silence Request requestBody: content: From a20b32a52bba9bea4c9351fbc02590bda44f7e10 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 21 Nov 2024 11:07:46 +0530 Subject: [PATCH 227/311] #OBS-I352 : label capitalization --- api-service/swagger-doc/openapi_v2.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 759c0997..60aaa6fe 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -19,10 +19,10 @@ tags: - name: Data Ingest - name: Data Query - name: Query Templates - - name: Alert Notification-channels + - name: Alert Notification Channels - name: Alert Silence - name: Alerts Wrapper - - name: Alert Metric_alias + - name: Alert Metric Alias paths: /v2/datasets/create: post: @@ -9536,7 +9536,7 @@ paths: /alerts/v1/notifications/search: post: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Search Notificaation Channels requestBody: content: {} @@ -9548,7 +9548,7 @@ paths: /alerts/v1/notifications/create: post: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Configure channel requestBody: content: @@ -9577,7 +9577,7 @@ paths: description: Unique identifier for the alert get: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Publish Channel responses: '200': @@ -9587,7 +9587,7 @@ paths: /alerts/v1/notifications/test: post: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Verify Configured Channel requestBody: content: @@ -9620,7 +9620,7 @@ paths: description: Unique identifier for the alert patch: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Update Specific Channel requestBody: content: @@ -9644,7 +9644,7 @@ paths: description: Unique identifier for the alert delete: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Delete Channel responses: '200': @@ -10023,7 +10023,7 @@ paths: /alerts/v1/metric/alias/create: post: tags: - - Alert Metric_alias + - Alert Metric Alias summary: Create Custom Metric requestBody: content: @@ -10042,7 +10042,7 @@ paths: /alerts/v1/metric/alias/search: post: tags: - - Alert Metric_alias + - Alert Metric Alias summary: List Create Metrics requestBody: content: @@ -10065,7 +10065,7 @@ paths: description: Unique identifier for the alert patch: tags: - - Alert Metric_alias + - Alert Metric Alias summary: Modify Metric requestBody: content: @@ -10089,7 +10089,7 @@ paths: description: Unique identifier for the alert delete: tags: - - Alert Metric_alias + - Alert Metric Alias summary: Delete Metric responses: '200': From b1edeb66d7aea95300cc1ffcc62cec82dc2a119b Mon Sep 17 00:00:00 2001 From: yashash <126703764+yashashkumar@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:10:15 +0530 Subject: [PATCH 228/311] Updated swagger doc (#284) * #OBS-I352 : updated swagger doc * #OBS-I352 : updated swagger doc * #OBS-I352 : added connector register api * #OBS-I352 : label capitalization --- api-service/swagger-doc/openapi_v2.yml | 170 +++++++++++++++++++------ 1 file changed, 132 insertions(+), 38 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 1f0c570f..60aaa6fe 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -10,24 +10,24 @@ servers: - url: http://localhost:3000 - url: http://{{host_ip}} tags: - - name: Dataset api's + - name: Dataset API's description: >- The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria. - - name: Connector api's + - name: Connector API's - name: Data Ingest - - name: Data query + - name: Data Query - name: Query Templates - - name: Alert Notification-channels - - name: Alert silence + - name: Alert Notification Channels + - name: Alert Silence - name: Alerts Wrapper - - name: Alert Metric_alias + - name: Alert Metric Alias paths: /v2/datasets/create: post: tags: - - Dataset api's + - Dataset API's summary: Dataset Create description: >- This API allows you to create new datasets used by the analytical data @@ -279,7 +279,7 @@ paths: example-0: summary: 'Failure: Master dataset already exists' value: - id: api.datasets.create + id: api.files.generate-url ver: v2 ts: '2024-07-16T08:37:28+05:30' params: @@ -294,7 +294,7 @@ paths: example-1: summary: 'Failure: Dataset already exists' value: - id: api.datasets.create + id: api.files.generate-url ver: v2 ts: '2024-07-16T08:38:05+05:30' params: @@ -309,7 +309,7 @@ paths: /v2/files/generate-url: post: tags: - - Dataset api's + - Dataset API's summary: Generate Presigned URLs description: This API generates presigned URLs to upload or download files from cloud requestBody: @@ -485,7 +485,7 @@ paths: /v2/datasets/update: patch: tags: - - Dataset api's + - Dataset API's summary: Update Dataset description: >- This API allows you to update existing datasets, add or remove denorm @@ -544,7 +544,6 @@ paths: - value: denorm_key: eid denorm_out_field: userdata - dataset_id: master-telemetry action: remove - value: denorm_key: eid @@ -840,7 +839,7 @@ paths: description: Unique identifier for the dataset get: tags: - - Dataset api's + - Dataset API's summary: Read Dataset description: >- This API allows you to read dataset from the requested dataset_id. User @@ -1112,7 +1111,7 @@ paths: /v2/datasets/list: post: tags: - - Dataset api's + - Dataset API's summary: List Datasets description: >- This API allows you to list all datasets. User can apply filters on @@ -3717,7 +3716,7 @@ paths: /v2/datasets/dataschema: post: tags: - - Dataset api's + - Dataset API's summary: Schema Generation description: This api is used to generate data schema for the given dataset event. requestBody: @@ -4619,7 +4618,7 @@ paths: /v2/datasets/status-transition: post: tags: - - Dataset api's + - Dataset API's summary: Dataset Status Transition description: >- This API allows you to perform status transition between 2 states. @@ -4870,7 +4869,7 @@ paths: /v2/datasets/import: post: tags: - - Dataset api's + - Dataset API's summary: Dataset import requestBody: content: @@ -5767,7 +5766,7 @@ paths: description: Unique identifier for the dataset get: tags: - - Dataset api's + - Dataset API's summary: Export Dataset parameters: - name: status @@ -6454,7 +6453,7 @@ paths: /v2/datasets/copy: post: tags: - - Dataset api's + - Dataset API's summary: Clone Dataset requestBody: content: @@ -6578,7 +6577,7 @@ paths: /v2/connectors/list: post: tags: - - Connector api's + - Connector API's summary: List Connectors requestBody: content: @@ -7126,7 +7125,7 @@ paths: description: Unique identifier for the connector get: tags: - - Connector api's + - Connector API's summary: Read Connector responses: '200': @@ -7412,6 +7411,101 @@ paths: error: code: CONNECTOR_NOT_FOUND message: 'Connector not found: postgres-conn' + /v2/connector/register: + post: + tags: + - Connector API's + summary: Register a connector + description: Registers a connector by uploading a file. + operationId: registerConnector + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + file3: + type: string + format: binary + description: Connector distribution file to upload. + responses: + '200': + description: Connector registered successfully. + content: + application/json: + schema: + type: object + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:28:47+00:00" + params: + type: object + properties: + status: + type: string + example: SUCCESS + resmsgid: + type: string + example: fc2ce24f-a333-479f-931c-024d61039801 + responseCode: + type: string + example: OK + result: + type: object + properties: + message: + type: string + example: connector registered successfully. + '500': + description: Internal server error while registering the connector. + content: + application/json: + schema: + type: object + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:50:21+00:00" + params: + type: object + properties: + status: + type: string + example: FAILED + resmsgid: + type: string + example: ebaad9d7-c0d4-47e4-8cde-87fb25eef739 + responseCode: + type: string + example: INTERNAL_SERVER_ERROR + error: + type: object + properties: + code: + type: string + example: FAILED_TO_REGISTER_CONNECTOR + message: + type: string + example: Failed to Extract the File + trace: + type: string + example: "" /v2/data/in/{dataset_id}: parameters: - name: dataset_id @@ -7645,7 +7739,7 @@ paths: /v2/data/query/test: post: tags: - - Data query + - Data Query summary: Query Data requestBody: content: @@ -9442,7 +9536,7 @@ paths: /alerts/v1/notifications/search: post: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Search Notificaation Channels requestBody: content: {} @@ -9454,7 +9548,7 @@ paths: /alerts/v1/notifications/create: post: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Configure channel requestBody: content: @@ -9483,7 +9577,7 @@ paths: description: Unique identifier for the alert get: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Publish Channel responses: '200': @@ -9493,7 +9587,7 @@ paths: /alerts/v1/notifications/test: post: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Verify Configured Channel requestBody: content: @@ -9526,7 +9620,7 @@ paths: description: Unique identifier for the alert patch: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Update Specific Channel requestBody: content: @@ -9550,7 +9644,7 @@ paths: description: Unique identifier for the alert delete: tags: - - Alert Notification-channels + - Alert Notification Channels summary: Delete Channel responses: '200': @@ -9560,7 +9654,7 @@ paths: /alerts/v1/silence/create: post: tags: - - Alert silence + - Alert Silence summary: Add Silence requestBody: content: @@ -9587,7 +9681,7 @@ paths: description: Unique identifier for the alert get: tags: - - Alert silence + - Alert Silence summary: Get Silence responses: '200': @@ -9604,7 +9698,7 @@ paths: description: Unique identifier for the alert delete: tags: - - Alert silence + - Alert Silence summary: Delete Silence responses: '200': @@ -9614,7 +9708,7 @@ paths: /alerts/v1/silence/search: get: tags: - - Alert silence + - Alert Silence summary: Fetch All Silences responses: '200': @@ -9631,7 +9725,7 @@ paths: description: Unique identifier for the alert patch: tags: - - Alert silence + - Alert Silence summary: Modify Silence Request requestBody: content: @@ -9929,7 +10023,7 @@ paths: /alerts/v1/metric/alias/create: post: tags: - - Alert Metric_alias + - Alert Metric Alias summary: Create Custom Metric requestBody: content: @@ -9948,7 +10042,7 @@ paths: /alerts/v1/metric/alias/search: post: tags: - - Alert Metric_alias + - Alert Metric Alias summary: List Create Metrics requestBody: content: @@ -9971,7 +10065,7 @@ paths: description: Unique identifier for the alert patch: tags: - - Alert Metric_alias + - Alert Metric Alias summary: Modify Metric requestBody: content: @@ -9995,10 +10089,10 @@ paths: description: Unique identifier for the alert delete: tags: - - Alert Metric_alias + - Alert Metric Alias summary: Delete Metric responses: '200': description: Successful response content: - application/json: {} \ No newline at end of file + application/json: {} From c8cfa803034978b6b0f4d9c00a94936d375f7093 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Thu, 21 Nov 2024 11:31:23 +0530 Subject: [PATCH 229/311] #000: Obsrv Installation Documentation - 1.2.0-RC --- api-service/swagger-doc/openapi_v2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 60aaa6fe..ba7bdca8 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -7736,7 +7736,7 @@ paths: code: TOPIC_NOT_FOUND message: Entry topic is not defined trace: '' - /v2/data/query/test: + /v2/data/query/{dataset_id}: post: tags: - Data Query @@ -7755,7 +7755,7 @@ paths: context: aggregationLevel: day query: >- - Select * from "check" WHERE __time >= timestamp '2020-12-31' + Select * from "check" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1 responses: '200': From 318fcd7d4f0e8ebf04716e5bb25b730008905606 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Thu, 21 Nov 2024 11:40:18 +0530 Subject: [PATCH 230/311] #000: Obsrv Installation Documentation - 1.2.0-RC --- api-service/swagger-doc/openapi_v2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index ba7bdca8..564ab32c 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -28,7 +28,7 @@ paths: post: tags: - Dataset API's - summary: Dataset Create + summary: Create Dataset description: >- This API allows you to create new datasets used by the analytical data source. @@ -7415,7 +7415,7 @@ paths: post: tags: - Connector API's - summary: Register a connector + summary: Register Connector description: Registers a connector by uploading a file. operationId: registerConnector requestBody: From 4a34787fe024e6f596fbfeb16d16e1389dfad5d2 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 14:41:03 +0530 Subject: [PATCH 231/311] #I250: Obsrv API Service: Generation of open telemetry log for the audit events --- api-service/package.json | 9 +-- api-service/src/app.ts | 86 +++------------------------ api-service/src/otel/OTelService.ts | 57 ++++++++---------- api-service/src/services/telemetry.ts | 2 + 4 files changed, 38 insertions(+), 116 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index b8f4548f..bf0a08fd 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -25,13 +25,14 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-logs-otlp-http": "^0.53.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.53.0", - "@opentelemetry/exporter-trace-otlp-http": "^0.53.0", - "@opentelemetry/resources": "^1.26.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", "@opentelemetry/sdk-logs": "^0.53.0", "@opentelemetry/sdk-metrics": "^1.26.0", "@opentelemetry/sdk-node": "^0.53.0", - "@opentelemetry/sdk-trace-node": "^1.26.0", - "@opentelemetry/semantic-conventions": "^1.27.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/sdk-trace-node": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index e61ea038..9e834d83 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -1,92 +1,20 @@ import express, { Application } from "express"; -import {router as v2Router} from "./routes/Router" -import { metricRouter } from "./routes/MetricRouter" -import { druidProxyRouter } from "./routes/DruidProxyRouter" +import { druidProxyRouter } from "./routes/DruidProxyRouter"; +import { metricRouter } from "./routes/MetricRouter"; +import { router as v2Router } from "./routes/Router"; import bodyParser from "body-parser"; -import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; -import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; +import { ResponseHandler } from "./helpers/ResponseHandler"; +import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; +import { OTelService } from "./otel/OTelService"; import { alertsRouter } from "./routes/AlertsRouter"; import { interceptAuditEvents } from "./services/telemetry"; -import { OTelService } from "./otel/OTelService"; -import { LogRecord } from "@opentelemetry/sdk-logs"; const app: Application = express(); -OTelService.init() -OTelService.log() - -const auditLog = { - "eid": "AUDIT", - "ets": 1729158293107, - "ver": "1.0.0", - "mid": "759b9471-b3bd-4818-89f5-cbdf5cdfc421", - "actor": { - "id": "SYSTEM", - "type": "User" - }, - "context": { - "env": "local", - "sid": "37229d4c-38a1-4aac-94cf-5fe34230fb1a", - "pdata": { - "id": "local.api.service", - "ver": "1.0" - } - }, - "object": {}, - "edata": { - "action": "dataset:create", - "props": [ - { - "property": "id", - "ov": null, - "nv": "api.data.in" - }, - { - "property": "ver", - "ov": null, - "nv": "v2" - }, - { - "property": "ts", - "ov": null, - "nv": "1711966306164" - }, - { - "property": "params", - "ov": null, - "nv": { - "msgid": "e180ecac-8f41-4f21-9a21-0b3a1a368917" - } - }, - { - "property": "data", - "ov": null, - "nv": { - "eid": "INTERACT", - "date": "2022-01-01", - "ver": "3.0", - "syncts": 1668591949682, - "ets": 1668591949682 - } - } - ], - "transition": { - "timeUnit": "ms", - "duration": 437, - "toState": "completed", - "fromState": "inprogress" - } - } -}; - -// Emit the audit log -OTelService.emitAuditLog(auditLog); -//const loggerInstance = OTelService.getLoggerProvider(); - - +OTelService.init() // Initialisation of Open telemetry Service. diff --git a/api-service/src/otel/OTelService.ts b/api-service/src/otel/OTelService.ts index 087381e6..77c960a1 100644 --- a/api-service/src/otel/OTelService.ts +++ b/api-service/src/otel/OTelService.ts @@ -1,15 +1,15 @@ import { Counter, diag, DiagConsoleLogger, DiagLogLevel, Meter, metrics } from '@opentelemetry/api'; -import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; -import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import * as logsAPI from '@opentelemetry/api-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; import { Resource } from '@opentelemetry/resources'; +import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { LoggerProvider, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'; -import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import logger from '../logger'; -import * as logsAPI from '@opentelemetry/api-logs'; export class OTelService { @@ -19,9 +19,9 @@ export class OTelService { public static init() { const collectorEndpoint = process.env.OTEL_COLLECTOR_ENDPOINT || 'http://localhost:4318'; - this.tracerProvider = this.createTracerProvider(collectorEndpoint); // Store the tracer provider - this.meterProvider = this.createMeterProvider(collectorEndpoint); // Store the meter provider - this.loggerProvider = this.createLoggerProvider(collectorEndpoint); // Store the logger provider + this.tracerProvider = this.createTracerProvider(collectorEndpoint); + this.meterProvider = this.createMeterProvider(collectorEndpoint); + this.loggerProvider = this.createLoggerProvider(collectorEndpoint); // Register the global tracer, meter, and logger providers this.tracerProvider.register(); @@ -123,41 +123,32 @@ export class OTelService { // Method to record the counter metric public static recordCounter(counter: Counter, value: number) { counter.add(value, { - // Optional attributes can be added here service: 'obsrv-api-service', }); } - // Method to log messages - public static log() { - const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); // Retrieve a logger instance - loggerInstance.emit({ - severityNumber: logsAPI.SeverityNumber.INFO, - severityText: 'INFO', - body: 'test', - attributes: { 'log.type': 'LogRecord' }, - }); - } - - public static emitAuditLog(auditLog: Record) { + public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); - // Construct the log record + const severityMapping: Record = { + INFO: logsAPI.SeverityNumber.INFO, + WARN: logsAPI.SeverityNumber.WARN, + ERROR: logsAPI.SeverityNumber.ERROR, + }; + + const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; + const logRecord = { - severityNumber: logsAPI.SeverityNumber.INFO, // or ERROR depending on the context - severityText: 'INFO', - body: JSON.stringify(auditLog), // Convert the log object to a string + severityNumber, + severityText: severity, + body: JSON.stringify(auditLog), attributes: { - 'log.type': 'AuditLog', - ...auditLog, // Include the whole log object as attributes if necessary + 'log.type': logType || 'console', + ...auditLog, }, }; - - // Emit the log record to OpenTelemetry loggerInstance.emit(logRecord); - - // Log the same message to Winston (optional) - logger.info("Audit log emitted", { auditLog }); } + } diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index 6f3a1670..380f35f1 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -3,6 +3,7 @@ import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import {send} from "../connections/kafkaConnection" +import { OTelService } from "../otel/OTelService"; const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); @@ -50,6 +51,7 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { + OTelService.generateOTelLog(event, 'INFO', 'audit-log'); send(event, telemetryTopic).catch(console.log); } From 5dd06bedfcad9fb7ec10a5be871a3114d60729fc Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 14:41:57 +0530 Subject: [PATCH 232/311] #I250: Removed the white spaces --- api-service/src/app.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 9e834d83..91b401bb 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -16,8 +16,6 @@ import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); OTelService.init() // Initialisation of Open telemetry Service. - - app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); From 0aa72c0a10326b82d140ca1e99b5c6b37d2f631b Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 14:43:23 +0530 Subject: [PATCH 233/311] #I250: Removed the hardcoded postgres details --- api-service/src/configs/ConnectionsConfig.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/configs/ConnectionsConfig.ts b/api-service/src/configs/ConnectionsConfig.ts index a12d0a56..b21512c9 100644 --- a/api-service/src/configs/ConnectionsConfig.ts +++ b/api-service/src/configs/ConnectionsConfig.ts @@ -5,8 +5,8 @@ export const connectionConfig = { postgres: { host: env.postgres_host || "localhost", port: env.postgres_port || 5432, - database: env.postgres_database || "postgres", - username: env.postgres_username || "manjunathdavanam", + database: env.postgres_database || "obsrv", + username: env.postgres_username || "postgres", password: env.postgres_password || "postgres", }, kafka: { From 61ab1a2be050b94322f403cf7ad41d551bd8b900 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 14:52:51 +0530 Subject: [PATCH 234/311] #I250: Variablised the open telemetry endpoint information --- api-service/src/configs/Config.ts | 1 + api-service/src/otel/OTelService.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index b26a6906..9ddb129d 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -115,4 +115,5 @@ export const config = { }, "user_token_public_key": process.env.user_token_public_key || "", "is_RBAC_enabled": process.env.is_rbac_enabled || "false", + "otel_collector_endpoint": process.env.OTEL_COLLECTOR_ENDPOINT || "http://localhost:4318" } diff --git a/api-service/src/otel/OTelService.ts b/api-service/src/otel/OTelService.ts index 77c960a1..87891111 100644 --- a/api-service/src/otel/OTelService.ts +++ b/api-service/src/otel/OTelService.ts @@ -10,7 +10,9 @@ import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import logger from '../logger'; - +import * as _ from "lodash"; +import { config } from "../configs/Config"; +const collectorEndpoint = _.get(config, "otel_collector_endpoint", "http://localhost:4318"); export class OTelService { private static meterProvider: MeterProvider; @@ -18,7 +20,6 @@ export class OTelService { private static tracerProvider: NodeTracerProvider; public static init() { - const collectorEndpoint = process.env.OTEL_COLLECTOR_ENDPOINT || 'http://localhost:4318'; this.tracerProvider = this.createTracerProvider(collectorEndpoint); this.meterProvider = this.createMeterProvider(collectorEndpoint); this.loggerProvider = this.createLoggerProvider(collectorEndpoint); From 452b81fae38e8763de55327047b7c65432884666 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 18:38:13 +0530 Subject: [PATCH 235/311] #I250: Moved the file from otel to services --- api-service/src/app.ts | 2 +- api-service/src/otel/OTelService.ts | 4 +- api-service/src/services/otel/OTelService.ts | 155 +++++++++++++++++++ api-service/src/services/telemetry.ts | 2 +- 4 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 api-service/src/services/otel/OTelService.ts diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 91b401bb..8982e594 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -7,7 +7,7 @@ import bodyParser from "body-parser"; import { config } from "./configs/Config"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; -import { OTelService } from "./otel/OTelService"; +import { OTelService } from "./services/otel/OTelService"; import { alertsRouter } from "./routes/AlertsRouter"; import { interceptAuditEvents } from "./services/telemetry"; diff --git a/api-service/src/otel/OTelService.ts b/api-service/src/otel/OTelService.ts index 87891111..2e4fed7d 100644 --- a/api-service/src/otel/OTelService.ts +++ b/api-service/src/otel/OTelService.ts @@ -9,9 +9,9 @@ import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; -import logger from '../logger'; +import logger from '../../logger'; import * as _ from "lodash"; -import { config } from "../configs/Config"; +import { config } from "../../configs/Config"; const collectorEndpoint = _.get(config, "otel_collector_endpoint", "http://localhost:4318"); export class OTelService { diff --git a/api-service/src/services/otel/OTelService.ts b/api-service/src/services/otel/OTelService.ts new file mode 100644 index 00000000..2e4fed7d --- /dev/null +++ b/api-service/src/services/otel/OTelService.ts @@ -0,0 +1,155 @@ +import { Counter, diag, DiagConsoleLogger, DiagLogLevel, Meter, metrics } from '@opentelemetry/api'; +import * as logsAPI from '@opentelemetry/api-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import logger from '../../logger'; +import * as _ from "lodash"; +import { config } from "../../configs/Config"; +const collectorEndpoint = _.get(config, "otel_collector_endpoint", "http://localhost:4318"); + +export class OTelService { + private static meterProvider: MeterProvider; + private static loggerProvider: LoggerProvider; + private static tracerProvider: NodeTracerProvider; + + public static init() { + this.tracerProvider = this.createTracerProvider(collectorEndpoint); + this.meterProvider = this.createMeterProvider(collectorEndpoint); + this.loggerProvider = this.createLoggerProvider(collectorEndpoint); + + // Register the global tracer, meter, and logger providers + this.tracerProvider.register(); + this.setGlobalMeterProvider(this.meterProvider); + + logger.info("OpenTelemetry Service Initialized"); + + // Add shutdown hook + process.on('SIGTERM', async () => { + await this.tracerProvider.shutdown(); + await this.meterProvider.shutdown(); + await this.loggerProvider.shutdown(); + }); + } + + private static createTracerProvider(endpoint: string) { + const traceExporter = new OTLPTraceExporter({ + url: `${endpoint}/v1/traces`, + }); + + const tracerProvider = new NodeTracerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + tracerProvider.addSpanProcessor(new BatchSpanProcessor(traceExporter)); + + return tracerProvider; + } + + private static createMeterProvider(endpoint: string) { + const metricExporter = new OTLPMetricExporter({ + url: `${endpoint}/v1/metrics`, + }); + + const meterProvider = new MeterProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + meterProvider.addMetricReader( + new PeriodicExportingMetricReader({ + exporter: metricExporter, + exportIntervalMillis: 10000, + }) + ); + + return meterProvider; + } + + private static createLoggerProvider(endpoint: string) { + const logExporter = new OTLPLogExporter({ + url: `${endpoint}/v1/logs`, + }); + + const loggerProvider = new LoggerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + loggerProvider.addLogRecordProcessor( + new BatchLogRecordProcessor(logExporter) + ); + + return loggerProvider; + } + + // Helper method to create a Resource with service name + private static createServiceResource(serviceName: string) { + return new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: serviceName, + }); + } + + private static setGlobalMeterProvider(meterProvider: MeterProvider) { + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); + diag.info('Registering MeterProvider globally.'); + metrics.setGlobalMeterProvider(meterProvider); + } + + // Method to create a counter metric + public static createCounterMetric(name: string): Counter { + const meter = this.getMeterProvider(); // Use the updated getMeterProvider method + const counter = meter.createCounter(name, { + description: 'Counts the number of API calls', + }); + return counter; + } + + public static getMeterProvider(): Meter { + return this.meterProvider.getMeter('obsrv-api-service'); + } + + public static getLoggerProvider(): LoggerProvider { + return this.loggerProvider; + } + + public static getTracerProvider(): NodeTracerProvider { + return this.tracerProvider; + } + + // Method to record the counter metric + public static recordCounter(counter: Counter, value: number) { + counter.add(value, { + service: 'obsrv-api-service', + }); + } + + + public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { + const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); + + const severityMapping: Record = { + INFO: logsAPI.SeverityNumber.INFO, + WARN: logsAPI.SeverityNumber.WARN, + ERROR: logsAPI.SeverityNumber.ERROR, + }; + + const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; + + const logRecord = { + severityNumber, + severityText: severity, + body: JSON.stringify(auditLog), + attributes: { + 'log.type': logType || 'console', + ...auditLog, + }, + }; + loggerInstance.emit(logRecord); + } + +} diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index 380f35f1..7aa4db86 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -3,7 +3,7 @@ import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import {send} from "../connections/kafkaConnection" -import { OTelService } from "../otel/OTelService"; +import { OTelService } from "./otel/OTelService"; const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); From 88f7fba1488003496123385bf89630a074dd1bbd Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 18:39:18 +0530 Subject: [PATCH 236/311] #I250: Moved the file from otel to services --- api-service/src/otel/OTelService.ts | 155 ---------------------------- 1 file changed, 155 deletions(-) delete mode 100644 api-service/src/otel/OTelService.ts diff --git a/api-service/src/otel/OTelService.ts b/api-service/src/otel/OTelService.ts deleted file mode 100644 index 2e4fed7d..00000000 --- a/api-service/src/otel/OTelService.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { Counter, diag, DiagConsoleLogger, DiagLogLevel, Meter, metrics } from '@opentelemetry/api'; -import * as logsAPI from '@opentelemetry/api-logs'; -import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; -import { Resource } from '@opentelemetry/resources'; -import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs'; -import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; -import logger from '../../logger'; -import * as _ from "lodash"; -import { config } from "../../configs/Config"; -const collectorEndpoint = _.get(config, "otel_collector_endpoint", "http://localhost:4318"); - -export class OTelService { - private static meterProvider: MeterProvider; - private static loggerProvider: LoggerProvider; - private static tracerProvider: NodeTracerProvider; - - public static init() { - this.tracerProvider = this.createTracerProvider(collectorEndpoint); - this.meterProvider = this.createMeterProvider(collectorEndpoint); - this.loggerProvider = this.createLoggerProvider(collectorEndpoint); - - // Register the global tracer, meter, and logger providers - this.tracerProvider.register(); - this.setGlobalMeterProvider(this.meterProvider); - - logger.info("OpenTelemetry Service Initialized"); - - // Add shutdown hook - process.on('SIGTERM', async () => { - await this.tracerProvider.shutdown(); - await this.meterProvider.shutdown(); - await this.loggerProvider.shutdown(); - }); - } - - private static createTracerProvider(endpoint: string) { - const traceExporter = new OTLPTraceExporter({ - url: `${endpoint}/v1/traces`, - }); - - const tracerProvider = new NodeTracerProvider({ - resource: this.createServiceResource('obsrv-api-service'), - }); - - tracerProvider.addSpanProcessor(new BatchSpanProcessor(traceExporter)); - - return tracerProvider; - } - - private static createMeterProvider(endpoint: string) { - const metricExporter = new OTLPMetricExporter({ - url: `${endpoint}/v1/metrics`, - }); - - const meterProvider = new MeterProvider({ - resource: this.createServiceResource('obsrv-api-service'), - }); - - meterProvider.addMetricReader( - new PeriodicExportingMetricReader({ - exporter: metricExporter, - exportIntervalMillis: 10000, - }) - ); - - return meterProvider; - } - - private static createLoggerProvider(endpoint: string) { - const logExporter = new OTLPLogExporter({ - url: `${endpoint}/v1/logs`, - }); - - const loggerProvider = new LoggerProvider({ - resource: this.createServiceResource('obsrv-api-service'), - }); - - loggerProvider.addLogRecordProcessor( - new BatchLogRecordProcessor(logExporter) - ); - - return loggerProvider; - } - - // Helper method to create a Resource with service name - private static createServiceResource(serviceName: string) { - return new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: serviceName, - }); - } - - private static setGlobalMeterProvider(meterProvider: MeterProvider) { - diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); - diag.info('Registering MeterProvider globally.'); - metrics.setGlobalMeterProvider(meterProvider); - } - - // Method to create a counter metric - public static createCounterMetric(name: string): Counter { - const meter = this.getMeterProvider(); // Use the updated getMeterProvider method - const counter = meter.createCounter(name, { - description: 'Counts the number of API calls', - }); - return counter; - } - - public static getMeterProvider(): Meter { - return this.meterProvider.getMeter('obsrv-api-service'); - } - - public static getLoggerProvider(): LoggerProvider { - return this.loggerProvider; - } - - public static getTracerProvider(): NodeTracerProvider { - return this.tracerProvider; - } - - // Method to record the counter metric - public static recordCounter(counter: Counter, value: number) { - counter.add(value, { - service: 'obsrv-api-service', - }); - } - - - public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { - const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); - - const severityMapping: Record = { - INFO: logsAPI.SeverityNumber.INFO, - WARN: logsAPI.SeverityNumber.WARN, - ERROR: logsAPI.SeverityNumber.ERROR, - }; - - const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; - - const logRecord = { - severityNumber, - severityText: severity, - body: JSON.stringify(auditLog), - attributes: { - 'log.type': logType || 'console', - ...auditLog, - }, - }; - loggerInstance.emit(logRecord); - } - -} From 33a1c69d669d5e8ac94b066d2ec17b55509cd36f Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 26 Nov 2024 18:45:33 +0530 Subject: [PATCH 237/311] #I250: Moved the file from otel to services --- api-service/src/app.ts | 2 +- api-service/src/configs/Config.ts | 7 ++++++- api-service/src/services/otel/OTelService.ts | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 8982e594..bfcc2f9a 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -14,7 +14,7 @@ import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); -OTelService.init() // Initialisation of Open telemetry Service. +(config.otel && config.otel.enable) && OTelService.init() // Initialisation of Open telemetry Service. app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 9ddb129d..f681e022 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -115,5 +115,10 @@ export const config = { }, "user_token_public_key": process.env.user_token_public_key || "", "is_RBAC_enabled": process.env.is_rbac_enabled || "false", - "otel_collector_endpoint": process.env.OTEL_COLLECTOR_ENDPOINT || "http://localhost:4318" + "otel": { + "enable": process.env.OTEL_ENABLE || true, + "collector_endpoint": process.env.OTEL_COLLECTOR_ENDPOINT || "http://localhost:4318" + } + + } diff --git a/api-service/src/services/otel/OTelService.ts b/api-service/src/services/otel/OTelService.ts index 2e4fed7d..fc8c0a54 100644 --- a/api-service/src/services/otel/OTelService.ts +++ b/api-service/src/services/otel/OTelService.ts @@ -12,7 +12,7 @@ import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' import logger from '../../logger'; import * as _ from "lodash"; import { config } from "../../configs/Config"; -const collectorEndpoint = _.get(config, "otel_collector_endpoint", "http://localhost:4318"); +const collectorEndpoint = _.get(config, "otel.collector_endpoint", "http://localhost:4318"); export class OTelService { private static meterProvider: MeterProvider; From 559b43914d7bf73ecc404f1993d6c5c8235448ee Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 27 Nov 2024 10:10:53 +0530 Subject: [PATCH 238/311] #I250: comparison fix --- api-service/src/app.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index bfcc2f9a..6a503114 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -10,11 +10,12 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { OTelService } from "./services/otel/OTelService"; import { alertsRouter } from "./routes/AlertsRouter"; import { interceptAuditEvents } from "./services/telemetry"; +import _ from "lodash"; const app: Application = express(); -(config.otel && config.otel.enable) && OTelService.init() // Initialisation of Open telemetry Service. +((config.otel && _.toLower(config.otel.enable) === "true")) && OTelService.init() // Initialisation of Open telemetry Service. app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); From 56f4cb4ac94cee4e3472c53f56712e76bf240511 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 27 Nov 2024 10:11:09 +0530 Subject: [PATCH 239/311] #I250: comparison fix --- api-service/src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 6a503114..e01491cd 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -15,7 +15,7 @@ import _ from "lodash"; const app: Application = express(); -((config.otel && _.toLower(config.otel.enable) === "true")) && OTelService.init() // Initialisation of Open telemetry Service. +((config.otel && _.toLower(config?.otel?.enable) === "true")) && OTelService.init() // Initialisation of Open telemetry Service. app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); From 640a8459f36ffc94b6c05ca39ecb3d3f7a7fb3a8 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 27 Nov 2024 10:11:45 +0530 Subject: [PATCH 240/311] #I250: config updated --- api-service/src/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index f681e022..61ca7980 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -116,7 +116,7 @@ export const config = { "user_token_public_key": process.env.user_token_public_key || "", "is_RBAC_enabled": process.env.is_rbac_enabled || "false", "otel": { - "enable": process.env.OTEL_ENABLE || true, + "enable": process.env.OTEL_ENABLE || "true", "collector_endpoint": process.env.OTEL_COLLECTOR_ENDPOINT || "http://localhost:4318" } From 35c437021d3ddca308c3f28bd3b326691413579f Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 27 Nov 2024 13:06:57 +0700 Subject: [PATCH 241/311] #OBS-I338 spark worker fixes (#286) --- .../spark-connector-cron/templates/cronjob.yaml | 2 +- .../helm-charts/spark-connector-cron/values.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index f22dcaef..7d35b32b 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -70,7 +70,7 @@ spec: - | # Wait for the Spark pod to be ready SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') - kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master=local[*] --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" {{- end }} {{- with .Values.sidecars }} {{- toYaml . | nindent 12 }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index ceca090a..7ed3ba10 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -22,10 +22,10 @@ commonAnnotations: {} podAnnotations: {} -podSecurityContext: {} - # runAsNonRoot: true - # runAsUser: 1001 - # fsGroup: 1001 +podSecurityContext: + runAsNonRoot: true + runAsUser: 1001 + fsGroup: 1001 securityContext: {} # readOnlyRootFilesystem: false From cc128fb6796a539e66b94c4317422b2d40938dba Mon Sep 17 00:00:00 2001 From: Manjunath Date: Thu, 28 Nov 2024 14:59:23 +0530 Subject: [PATCH 242/311] #I250: Updated the configurations of open telemetry service --- api-service/src/app.ts | 3 +- api-service/src/configs/Config.ts | 4 +- api-service/src/services/otel/OTelService.ts | 43 +++++++++++--------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index bfcc2f9a..101a4d9e 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -14,7 +14,8 @@ import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); -(config.otel && config.otel.enable) && OTelService.init() // Initialisation of Open telemetry Service. +// Initialisation of Open telemetry Service. +(config.otel && config.otel.enable) ? OTelService.init() : console.log("OpenTelemetry Service is disabled"); app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index f681e022..6a9cf1e5 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -116,9 +116,7 @@ export const config = { "user_token_public_key": process.env.user_token_public_key || "", "is_RBAC_enabled": process.env.is_rbac_enabled || "false", "otel": { - "enable": process.env.OTEL_ENABLE || true, + "enable": process.env.OTEL_ENABLE || false, "collector_endpoint": process.env.OTEL_COLLECTOR_ENDPOINT || "http://localhost:4318" } - - } diff --git a/api-service/src/services/otel/OTelService.ts b/api-service/src/services/otel/OTelService.ts index fc8c0a54..aa56425a 100644 --- a/api-service/src/services/otel/OTelService.ts +++ b/api-service/src/services/otel/OTelService.ts @@ -130,26 +130,29 @@ export class OTelService { public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { - const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); - - const severityMapping: Record = { - INFO: logsAPI.SeverityNumber.INFO, - WARN: logsAPI.SeverityNumber.WARN, - ERROR: logsAPI.SeverityNumber.ERROR, - }; - - const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; - - const logRecord = { - severityNumber, - severityText: severity, - body: JSON.stringify(auditLog), - attributes: { - 'log.type': logType || 'console', - ...auditLog, - }, - }; - loggerInstance.emit(logRecord); + if(config.otel && config.otel.enable){ + const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); + + const severityMapping: Record = { + INFO: logsAPI.SeverityNumber.INFO, + WARN: logsAPI.SeverityNumber.WARN, + ERROR: logsAPI.SeverityNumber.ERROR, + }; + + const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; + + const logRecord = { + severityNumber, + severityText: severity, + body: JSON.stringify(auditLog), + attributes: { + 'log.type': logType || 'console', + ...auditLog, + }, + }; + loggerInstance.emit(logRecord); + } } + } From 4ad396bf9fc81af219f6b140c8de7a4be63f0887 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Thu, 28 Nov 2024 16:47:18 +0530 Subject: [PATCH 243/311] #I250: Moved the file from otel to services --- api-service/src/services/otel/OTelService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/services/otel/OTelService.ts b/api-service/src/services/otel/OTelService.ts index aa56425a..65186cbd 100644 --- a/api-service/src/services/otel/OTelService.ts +++ b/api-service/src/services/otel/OTelService.ts @@ -130,7 +130,7 @@ export class OTelService { public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { - if(config.otel && config.otel.enable){ + if((config.otel && _.toLower(config?.otel?.enable) === "true")){ const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); const severityMapping: Record = { From 397922d50a15c044be3ddc8301edd1150a2a3911 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Dec 2024 02:18:11 +0530 Subject: [PATCH 244/311] #OBS-I339: updated rbac algorithm and bearer token --- api-service/src/middlewares/RBAC_middleware.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 10fda3e2..bcc9bcd2 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -77,7 +77,7 @@ const checkAccess = (decoded: any, action: string, req: Request, res: Response) const basicToken = (token: string, req: Request, res: Response, next: NextFunction) => { try { - const decoded = jwt.verify(token, config.user_token_public_key); + const decoded = jwt.verify(token, config.user_token_public_key, { algorithms: ['RS256'] }); if (!decoded || !_.isObject(decoded)) { return errorHandler(401, "Token verification failed or invalid token", req, res); @@ -125,8 +125,8 @@ export default { (req as any).userID = "SYSTEM"; return next(); } - - const token = req.get("x-user-token"); + const authHeader = req.headers['authorization']; + const token = authHeader && authHeader.split(' ')[1]; if (!token) { return errorHandler(401, "No token provided", req, res); } From 8b194e3d8acdf68fa4b63f30ddcb58530b9344c4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Dec 2024 02:18:56 +0530 Subject: [PATCH 245/311] #OBS-I339: added druid proxy --- api-service/src/routes/DruidProxyRouter.ts | 3 ++- api-service/src/services/WrapperService.ts | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index fdf73315..ac45710c 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -20,4 +20,5 @@ druidProxyRouter.post(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.n druidProxyRouter.get(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet); druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) +druidProxyRouter.get(/\/druid\/coordinator(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({entity: Entity.DruidProxy}), wrapperService.forwardNativeGetDatasource) \ No newline at end of file diff --git a/api-service/src/services/WrapperService.ts b/api-service/src/services/WrapperService.ts index c38afb6d..8742f931 100644 --- a/api-service/src/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -85,6 +85,22 @@ class WrapperService { } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } }; + public forwardNativeGetDatasource = async ( + req: Request, + res: Response, + next: NextFunction + ) => { + try { + const headers = req?.headers; + const url = req?.url; + const result = await axios.get( + `${config.query_api.druid.host}:${config.query_api.druid.port}${url}`, + { headers } + ); + ResponseHandler.flatResponse(req, res, result); + } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } + }; + public nativeStatus = async ( req: Request, res: Response, From 38b311d1f67ebd7931961c41336ef2ac0332f5b1 Mon Sep 17 00:00:00 2001 From: Manjunath Davanam Date: Mon, 2 Dec 2024 11:42:11 +0530 Subject: [PATCH 246/311] #I250: Following the strandrads in the configurations (#288) Co-authored-by: Manjunath --- api-service/src/configs/Config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 3539f491..51e0a7ab 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -116,7 +116,7 @@ export const config = { "user_token_public_key": process.env.user_token_public_key || "", "is_RBAC_enabled": process.env.is_rbac_enabled || "false", "otel": { - "enable": process.env.OTEL_ENABLE || "false", - "collector_endpoint": process.env.OTEL_COLLECTOR_ENDPOINT || "http://localhost:4318" + "enable": process.env.otel_enable || "false", + "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" } } From 255ffbf71b603669858986894563294e1dd9303b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Dec 2024 12:28:31 +0530 Subject: [PATCH 247/311] #OBS-I339: handling object type --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index d428b02a..d8aed3c3 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -94,6 +94,10 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { + if (!Array.isArray(connectorsConfig)) { + return []; + } + return connectorsConfig.map((connector: any) => { let connector_config = _.get(connector, "connector_config"); const authMechanism = _.get(connector_config, ["authenticationMechanism"]); From 1cc369907947518cb480ea61c314e2c5f6b73942 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Dec 2024 12:43:55 +0530 Subject: [PATCH 248/311] #OBS-I339: handling object removed --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index d8aed3c3..991cb2e3 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -94,9 +94,6 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { - if (!Array.isArray(connectorsConfig)) { - return []; - } return connectorsConfig.map((connector: any) => { let connector_config = _.get(connector, "connector_config"); From a6fea9b9302a1c20475f0c940392f6f0fee27f52 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Dec 2024 14:56:40 +0530 Subject: [PATCH 249/311] #OBS-I339: removed space --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 991cb2e3..d428b02a 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -94,7 +94,6 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { - return connectorsConfig.map((connector: any) => { let connector_config = _.get(connector, "connector_config"); const authMechanism = _.get(connector_config, ["authenticationMechanism"]); From e9a35f19af5a6405e5060a2bfd10cf21164c6d92 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Dec 2024 11:11:26 +0530 Subject: [PATCH 250/311] #OBS-I375: fix: fixing the expression field having spaces within it in the ingestion schema. --- api-service/src/services/TableGenerator.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index 7030ad95..270bfec1 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -58,7 +58,7 @@ class BaseTableGenerator { const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); - _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + `['${denormField.denorm_out_field}']`)); return prop; }); dataFields.push(...transformProps); @@ -66,7 +66,7 @@ class BaseTableGenerator { } if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ - expr: "$." + tf.field_key, + expr: "$." + `['${tf.field_key}']`, name: tf.field_key, data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, @@ -90,7 +90,7 @@ class BaseTableGenerator { const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([_.replace(denormField.denorm_out_field, /\./g, "_"), prop.name], "_")); - _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + `['${denormField.denorm_out_field}']`)); return prop; }); dataFields.push(...transformProps); @@ -98,7 +98,7 @@ class BaseTableGenerator { } if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ - expr: "$." + tf.field_key, + expr: "$." + `['${tf.field_key}']`, name: _.replace(tf.field_key, /\./g, "_"), data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, From 509f88d35814f6156897f3e4dd7d774bd8ae8f3b Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Dec 2024 11:37:13 +0530 Subject: [PATCH 251/311] #OBS-I375: fix: reverting changes for hudi spec --- api-service/src/services/TableGenerator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index 270bfec1..661a3919 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -90,7 +90,7 @@ class BaseTableGenerator { const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([_.replace(denormField.denorm_out_field, /\./g, "_"), prop.name], "_")); - _.set(prop, "expr", _.replace(prop.expr, "$", "$." + `['${denormField.denorm_out_field}']`)); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); return prop; }); dataFields.push(...transformProps); @@ -98,7 +98,7 @@ class BaseTableGenerator { } if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ - expr: "$." + `['${tf.field_key}']`, + expr: "$." + tf.field_key, name: _.replace(tf.field_key, /\./g, "_"), data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, From 3ed101885d13709a0719ba738e3fd135b9bf0af7 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Dec 2024 13:18:51 +0530 Subject: [PATCH 252/311] #OBS-I369: fix: aws storage service changes --- api-service/src/services/CloudServices/AWSStorageService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api-service/src/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts index d83b65b1..61e6f627 100644 --- a/api-service/src/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -16,10 +16,14 @@ export class AWSStorageService implements ICloudService { const accessKeyId = _.get(config, "identity") const secretAccessKey = _.get(config, "credential") const endpoint = _.get(config, "endpoint") + const s3ForcePathStyle = _.get(config, "s3ForcePathStyle") const configuration: any = { region, credentials: { accessKeyId, secretAccessKey } } if(endpoint) { configuration.endpoint = endpoint; } + if (s3ForcePathStyle) { + configuration.s3ForcePathStyle = s3ForcePathStyle; + } try { this.client = new S3Client(configuration); } From 1e23bd1fbe5005948904b63deeb92faca7cbcb97 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Mon, 16 Dec 2024 17:35:23 +0530 Subject: [PATCH 253/311] Vulnerability fixes (#292) * #I431: Vuln fix with NodeJs 23 * #I431: ignored postman collection * #I431: ignored postman collection * #I431: removed postman collection --- api-service/.dockerignore | 4 +++- api-service/Dockerfile | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api-service/.dockerignore b/api-service/.dockerignore index b512c09d..b83f1fd0 100644 --- a/api-service/.dockerignore +++ b/api-service/.dockerignore @@ -1 +1,3 @@ -node_modules \ No newline at end of file +node_modules +api-service/postman-collection +postman-collection \ No newline at end of file diff --git a/api-service/Dockerfile b/api-service/Dockerfile index 2d0f168d..76e7174f 100644 --- a/api-service/Dockerfile +++ b/api-service/Dockerfile @@ -1,7 +1,8 @@ -FROM --platform=linux/amd64 node:20-alpine +FROM --platform=linux/amd64 node:23.4-alpine RUN mkdir -p /opt/api-service COPY ./api-service ./opt/api-service WORKDIR /opt/api-service +RUN rm -rf postman-collection RUN npm install COPY . . EXPOSE 3000 From 90a024b3840895446c04cd0be5aaeddf7dcb2c64 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Dec 2024 18:41:59 +0530 Subject: [PATCH 254/311] #OBS-I410: updated list api to return connectors_config and version_key for draft dataset --- api-service/src/controllers/DatasetList/DatasetList.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts index 82acbf62..c11b3d37 100644 --- a/api-service/src/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -12,8 +12,7 @@ export const apiId = "api.datasets.list" export const errorCode = "DATASET_LIST_FAILURE" const liveDatasetStatus = ["Live", "Retired", "Purged"] const draftDatasetStatus = ["Draft", "ReadyToPublish"] -const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date"] - +const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date", "data_schema", "validation_config", "dedup_config", "denorm_config"] const datasetList = async (req: Request, res: Response) => { const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) @@ -37,7 +36,7 @@ const listDatasets = async (request: Record): Promise Date: Wed, 18 Dec 2024 10:43:56 +0530 Subject: [PATCH 255/311] #OBS-I410: added requierd fields for draft datasets --- api-service/src/controllers/DatasetList/DatasetList.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts index c11b3d37..986cd50e 100644 --- a/api-service/src/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -12,7 +12,7 @@ export const apiId = "api.datasets.list" export const errorCode = "DATASET_LIST_FAILURE" const liveDatasetStatus = ["Live", "Retired", "Purged"] const draftDatasetStatus = ["Draft", "ReadyToPublish"] -const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date", "data_schema", "validation_config", "dedup_config", "denorm_config"] +const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date"] const datasetList = async (req: Request, res: Response) => { const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) @@ -36,7 +36,7 @@ const listDatasets = async (request: Record): Promise Date: Wed, 18 Dec 2024 10:45:21 +0530 Subject: [PATCH 256/311] #OBS-I410: formatted --- api-service/src/controllers/DatasetList/DatasetList.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts index 986cd50e..6cad65dc 100644 --- a/api-service/src/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -13,6 +13,7 @@ export const errorCode = "DATASET_LIST_FAILURE" const liveDatasetStatus = ["Live", "Retired", "Purged"] const draftDatasetStatus = ["Draft", "ReadyToPublish"] const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date"] + const datasetList = async (req: Request, res: Response) => { const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) From ecffa25860d0e8411b5812b8708ca2a543dcbcb9 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Thu, 19 Dec 2024 11:54:09 +0530 Subject: [PATCH 257/311] fix: updated configuration for aws storage service --- api-service/src/services/CloudServices/AWSStorageService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts index 61e6f627..53a6679d 100644 --- a/api-service/src/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -22,7 +22,7 @@ export class AWSStorageService implements ICloudService { configuration.endpoint = endpoint; } if (s3ForcePathStyle) { - configuration.s3ForcePathStyle = s3ForcePathStyle; + configuration.forcePathStyle = s3ForcePathStyle; } try { this.client = new S3Client(configuration); From ada75814686020725ad3ea02ad8de3f1b4197c18 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Thu, 19 Dec 2024 12:51:57 +0530 Subject: [PATCH 258/311] #OBS-I418 update ns for flink connectors --- command-service/helm-charts/flink-connector/values.yaml | 2 +- command-service/src/config/service_config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index ed51316d..3076a343 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -4,7 +4,7 @@ fullnameOverride: "" replicaCount: 1 -namespace: "flink" +namespace: "flink-connectors" commonLabels: system.processing: "true" release: monitoring diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 80276238..89cfa98e 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -175,7 +175,7 @@ connector_jobs: namespace: spark base_helm_chart: spark-connector-cron flink: - namespace: flink + namespace: flink-connectors base_helm_chart: flink-connector connector_registry: From ae4824c6d001af0f21d0d51afe69eaf910289506 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Tue, 24 Dec 2024 11:45:24 +0530 Subject: [PATCH 259/311] #OBS-I441 fix spark connector deployments --- .../flink-connector/templates/deployment.yaml | 5 +- .../helm-charts/flink-connector/values.yaml | 2 + .../templates/cronjob.yaml | 6 +- .../spark-connector-cron/values.yaml | 3 + .../src/command/connector_command.py | 106 +++++++++--------- .../src/command/connector_registry.py | 8 +- 6 files changed, 70 insertions(+), 60 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 74fc9cca..99ceee79 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -104,9 +104,10 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} - component: {{ printf "%s-%s" $jobName $component }} + dataset: {{ .Values.dataset_id }} + connector: {{ .Values.connector_id }} annotations: - checksum/config: {{ .Files.Glob "cFonfigs/*" | toYaml | sha256sum }} + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} spec: serviceAccountName: {{ include "base.serviceaccountname" . }} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index 3076a343..fd9dcda8 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -246,6 +246,8 @@ serviceMonitor: # override flink_jobs # flink_jobs: +dataset_id: "" +connector_id: "" commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index 7d35b32b..a80831fc 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -1,9 +1,11 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "base.cronReleaseName" . }} + name: {{ .Release.Name }} namespace: {{ include "base.namespace" . }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/dataset: {{ .Values.dataset_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} {{- if .Values.commonAnnotations }} annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} @@ -19,6 +21,8 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 12 }} {{- end }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} + app.kubernetes.io/dataset: {{ .Values.dataset_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} spec: serviceAccountName: {{ .Values.serviceAccount.name }} restartPolicy: {{ .Values.restartPolicy }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index 7ed3ba10..8069fe0e 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -142,6 +142,9 @@ instance_id: nyt-psql.1 main_class: org.sunbird.obsrv.connector.JDBCConnector main_file: jdbc-connector-1.0.0.jar +dataset_id: "" +connector_id: "" + ## Object Store Connector # technology: python # connector-source: object_store_connector-0.1.0 diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 8f5b145e..f91313c4 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -8,6 +8,7 @@ from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType from model.db_models import ConnectorInstance from service.db_service import DatabaseService +from datetime import datetime class ConnectorCommand(ICommand): @@ -47,6 +48,7 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data print(f"Uninstalling jobs for {namespace}..") base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] + # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -57,6 +59,40 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data # for release in masterdata_jar_config: # managed_releases.append(release["release_name"]) + deployed_cron_jobs_cmd = [ + "kubectl", "get", "cronjobs", "-n", namespace, + "--selector", f"app.kubernetes.io/dataset={dataset_id}", + "-o", "jsonpath=\"{.items[*].metadata.name}\"" + ] + print(" ".join(deployed_cron_jobs_cmd)) + + deployed_cron_jobs_result = subprocess.run( + deployed_cron_jobs_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + if deployed_cron_jobs_result.returncode == 0 and len(deployed_cron_jobs_result.stdout.decode().replace("\"", "").splitlines()): + jobs = deployed_cron_jobs_result.stdout.decode().replace("\"", "").splitlines()[0].split() + for job in jobs: + print(f"Uninstalling job {job} related to dataset'{dataset_id}'...") + helm_uninstall_cmd = [ + "helm", + "uninstall", + job, + "--namespace", + namespace, + ] + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled job '{job}'...") + else: + print(f"Error uninstalling job '{job}': {helm_uninstall_result.stderr.decode()}") + + ## TODO: Add flink uninstall logic helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -67,7 +103,7 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data spark_connector = {connector.id for connector in active_connectors if connector.connector_runtime == "spark"} for release_name in spark_connector: if release_name in job_names: - print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") + print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") helm_uninstall_cmd = [ "helm", "uninstall", @@ -75,6 +111,7 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data "--namespace", namespace, ] + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) helm_uninstall_result = subprocess.run( helm_uninstall_cmd, stdout=subprocess.PIPE, @@ -84,7 +121,7 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data print(f"Successfully uninstalled job '{release_name}'...") else: print(f"Error uninstalling job '{release_name}': {helm_uninstall_result.stderr.decode()}") - + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None for connector in active_connectors: @@ -125,7 +162,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: - restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,app.kubernetes.io/component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,app.kubernetes.io/component={job_name}-taskmanager --namespace {namespace}".format( namespace=namespace, job_name=job_name ) print("Restart command: ", restart_cmd) @@ -140,12 +177,12 @@ def _perform_flink_install(self, dataset_id, connector_instance): print(f"Job {job_name} restart succeeded...") else: err = True + print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") return ActionResponse( status="ERROR", status_code=500, error_message="FLINK_HELM_LIST_EXCEPTION", ) - print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") if err is None: result = ActionResponse(status="OK", status_code=200) @@ -172,6 +209,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): "--namespace", namespace, "--create-namespace", + "--set", "namespace={}".format(namespace), "--set-json", f"""flink_jobs={set_json_value.replace(" ", "")}""" ] @@ -202,7 +240,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): return result else: - self._stop_connector_jobs(is_masterdata=False, namespace="flink") + self._stop_connector_jobs(is_masterdata=False, namespace=self.connector_job_config["flink"]["namespace"], active_connectors=[connector_instance], dataset_id=dataset_id) else: print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") return ActionResponse( @@ -214,7 +252,11 @@ def _perform_flink_install(self, dataset_id, connector_instance): def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None - release_name = connector_instance.id + release_name = "{}-{}-{}".format( + dataset_id[:8].lower().replace("_", "-"), + connector_instance.connector_id[:8].lower().replace("_", "-"), + datetime.now().strftime("%Y%m%d%H%M%S") + ) connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] @@ -239,7 +281,11 @@ def _perform_spark_install(self, dataset_id, connector_instance): "--set", "technology={}".format(connector_instance.technology), "--set", - "instance_id={}".format(release_name), + "dataset_id={}".format(dataset_id), + "--set", + "connector_id={}".format(connector_instance.connector_id), + "--set", + "instance_id={}".format(connector_instance.id), "--set", "connector_source={}".format(connector_source["source"]), "--set", @@ -320,49 +366,3 @@ def _get_live_instances(self, runtime, connector_instance): has_live_instances = True return has_live_instances - - # def _perform_install(self, release): - # err = None - # result = None - # release_name = release["release_name"] - # helm_install_cmd = [ - # "helm", - # "upgrade", - # "--install", - # release_name, - # self.connector_job_chart_dir, - # "--namespace", - # self.connector_job_ns, - # "--create-namespace", - # "--set", - # "file.path={}".format(release["jar"]), - # "--set", - # "class.name={}".format(release["class"]), - # "--set", - # "job.name={}".format(release_name), - # "--set", - # "args={}".format(",".join(release["args"])), - # "--set", - # "schedule={}".format(release["schedule"]), - # ] - # helm_install_result = subprocess.run( - # helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - # ) - # if helm_install_result.returncode == 0: - # print(f"Job {release_name} deployment succeeded...") - # else: - # err = True - # result = ActionResponse( - # status="ERROR", - # status_code=500, - # error_message="FLINK_HELM_INSTALLATION_EXCEPTION", - # ) - # print( - # f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" - # ) - - # if err is None: - # result = ActionResponse(status="OK", status_code=200) - - # return result - diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 40edf5f7..140d59bc 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -214,9 +214,6 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) - subprocess.run(["rm", "-rf", self.extraction_path]) - subprocess.run(["rm", "-rf", self.download_path]) - if not success: return RegistryResponse( status="failure", @@ -224,13 +221,16 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) result.append(registry_meta.to_dict()) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + return RegistryResponse( status="success", connector_info=result, message="Connectors registered successfully", statusCode=status.HTTP_200_OK ) - else: connector_id = ( self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") From 6bd53a940cab79b4b19327ca105e5cd8a8ebcfae Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 24 Dec 2024 17:07:47 +0530 Subject: [PATCH 260/311] #OBS-I441: replcae datetime with time module --- command-service/src/command/connector_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index f91313c4..8e3ac5bf 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -8,7 +8,7 @@ from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType from model.db_models import ConnectorInstance from service.db_service import DatabaseService -from datetime import datetime +import time class ConnectorCommand(ICommand): @@ -255,7 +255,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): release_name = "{}-{}-{}".format( dataset_id[:8].lower().replace("_", "-"), connector_instance.connector_id[:8].lower().replace("_", "-"), - datetime.now().strftime("%Y%m%d%H%M%S") + str(time.time_ns()) ) connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] From 9fa14dd1fd1f0a7f915085bcd345770c7fcef3b5 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Thu, 26 Dec 2024 10:41:44 +0530 Subject: [PATCH 261/311] #OBS-I441 fix flink connector deployments --- .../flink-connector/templates/deployment.yaml | 6 +- .../helm-charts/flink-connector/values.yaml | 1 - .../src/command/connector_command.py | 89 ++++++++++--------- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 99ceee79..9e2c9802 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -10,6 +10,7 @@ metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" $ }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} @@ -27,6 +28,7 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + app.kubernetes.io/connector: {{ .Values.connector_id }} component: {{ printf "%s-%s" $jobName $component }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} @@ -87,6 +89,7 @@ metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" . }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} @@ -104,8 +107,7 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} - dataset: {{ .Values.dataset_id }} - connector: {{ .Values.connector_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index fd9dcda8..ccabbc54 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -246,7 +246,6 @@ serviceMonitor: # override flink_jobs # flink_jobs: -dataset_id: "" connector_id: "" commonAnnotations: diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 8e3ac5bf..4a92ee17 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -39,16 +39,12 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"], active_connectors, dataset_id) + self._stop_connector_jobs(is_masterdata, dataset_id) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result - def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, dataset_id): - print(f"Uninstalling jobs for {namespace}..") - base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - - + def _stop_connector_jobs(self, is_masterdata, dataset_id): # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -59,12 +55,13 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data # for release in masterdata_jar_config: # managed_releases.append(release["release_name"]) + ## Clear all spark cronjobs for the dataset + spark_namespace = self.connector_job_config["spark"]["namespace"] deployed_cron_jobs_cmd = [ - "kubectl", "get", "cronjobs", "-n", namespace, + "kubectl", "get", "cronjobs", "-n", spark_namespace, "--selector", f"app.kubernetes.io/dataset={dataset_id}", "-o", "jsonpath=\"{.items[*].metadata.name}\"" ] - print(" ".join(deployed_cron_jobs_cmd)) deployed_cron_jobs_result = subprocess.run( deployed_cron_jobs_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -79,7 +76,7 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data "uninstall", job, "--namespace", - namespace, + spark_namespace, ] print("Uninstall command: ", " ".join(helm_uninstall_cmd)) helm_uninstall_result = subprocess.run( @@ -92,35 +89,31 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data else: print(f"Error uninstalling job '{job}': {helm_uninstall_result.stderr.decode()}") - ## TODO: Add flink uninstall logic - helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - helm_ls_result = subprocess.run( - helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode().splitlines()[1:] - job_names = {job.split()[0] for job in jobs if base_helm_chart in job} - spark_connector = {connector.id for connector in active_connectors if connector.connector_runtime == "spark"} - for release_name in spark_connector: - if release_name in job_names: - print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") - helm_uninstall_cmd = [ - "helm", - "uninstall", - release_name, - "--namespace", - namespace, - ] - print("Uninstall command: ", " ".join(helm_uninstall_cmd)) - helm_uninstall_result = subprocess.run( - helm_uninstall_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if helm_uninstall_result.returncode == 0: - print(f"Successfully uninstalled job '{release_name}'...") - else: - print(f"Error uninstalling job '{release_name}': {helm_uninstall_result.stderr.decode()}") + ## Clear all flink connectors that are not active + flink_namespace = self.connector_job_config["flink"]["namespace"] + registered_connectors = self._get_registered_connectors(runtime="flink") + print("Registered flink connectors: ", registered_connectors) + for connector in registered_connectors: + if connector["instance_count"] == 0: + ### Uninstall the helm chart + helm_uninstall_cmd = [ + "helm", + "uninstall", + connector["id"].replace(".", "-").replace("_", "-"), + "--namespace", + flink_namespace, + ] + + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled deployment '{connector['id']}'...") + else: + print(f"Error uninstalling deployment '{connector['id']}': {helm_uninstall_result.stderr.decode()}") def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None @@ -167,13 +160,13 @@ def _perform_flink_install(self, dataset_id, connector_instance): ) print("Restart command: ", restart_cmd) # Run the helm command - helm_install_result = subprocess.run( + restart_cmd_result = subprocess.run( restart_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, ) - if helm_install_result.returncode == 0: + if restart_cmd_result.returncode == 0: print(f"Job {job_name} restart succeeded...") else: err = True @@ -210,6 +203,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set", "namespace={}".format(namespace), + "--set", "connector_id={}".format(connector_instance.connector_id), "--set-json", f"""flink_jobs={set_json_value.replace(" ", "")}""" ] @@ -239,8 +233,6 @@ def _perform_flink_install(self, dataset_id, connector_instance): result = ActionResponse(status="OK", status_code=200) return result - else: - self._stop_connector_jobs(is_masterdata=False, namespace=self.connector_job_config["flink"]["namespace"], active_connectors=[connector_instance], dataset_id=dataset_id) else: print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") return ActionResponse( @@ -350,7 +342,6 @@ def _get_masterdata_details(self, dataset_id): return is_masterdata - ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False query = f""" @@ -366,3 +357,15 @@ def _get_live_instances(self, runtime, connector_instance): has_live_instances = True return has_live_instances + + def _get_registered_connectors(self, runtime): + query = f""" + SELECT cr.id, COUNT(ci.id) AS instance_count + FROM connector_registry cr + LEFT JOIN connector_instances ci ON cr.id = ci.connector_id + WHERE cr.runtime = %s + GROUP BY cr.id + """ + params = (runtime,) + rows = self.db_service.execute_select_all(sql=query, params=params) + return rows \ No newline at end of file From ef8fa4f214e8d566420a5a1be005745269fe437c Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Fri, 27 Dec 2024 10:56:46 +0530 Subject: [PATCH 262/311] #OBS-I442 Removed env for topics --- api-service/src/configs/Config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 51e0a7ab..cd74c111 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -42,8 +42,8 @@ export const config = { "connectionTimeout": process.env.kafka_connection_timeout ? parseInt(process.env.kafka_connection_timeout) : 5000 }, "topics": { // Default Kafka topics depend on type of dataset. - "createDataset": `${process.env.system_env || "local"}.ingest`, - "createMasterDataset": `${process.env.system_env || "local"}.masterdata.ingest` + "createDataset": `ingest`, + "createMasterDataset": `masterdata.ingest` } } }, From f12ef8dbb81098d36dcd331d133313f319e0d8ab Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Mon, 30 Dec 2024 14:44:35 +0530 Subject: [PATCH 263/311] #OBS-I441 update deployment of connectors --- .../src/command/connector_command.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 4a92ee17..b5073665 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -9,6 +9,8 @@ from model.db_models import ConnectorInstance from service.db_service import DatabaseService import time +from random import choice +from string import ascii_lowercase class ConnectorCommand(ICommand): @@ -94,12 +96,12 @@ def _stop_connector_jobs(self, is_masterdata, dataset_id): registered_connectors = self._get_registered_connectors(runtime="flink") print("Registered flink connectors: ", registered_connectors) for connector in registered_connectors: - if connector["instance_count"] == 0: + if connector[1] == 0: ### Uninstall the helm chart helm_uninstall_cmd = [ "helm", "uninstall", - connector["id"].replace(".", "-").replace("_", "-"), + connector[0].replace(".", "-").replace("_", "-"), "--namespace", flink_namespace, ] @@ -111,9 +113,9 @@ def _stop_connector_jobs(self, is_masterdata, dataset_id): stderr=subprocess.PIPE, ) if helm_uninstall_result.returncode == 0: - print(f"Successfully uninstalled deployment '{connector['id']}'...") + print(f"Successfully uninstalled deployment '{connector[0]}'...") else: - print(f"Error uninstalling deployment '{connector['id']}': {helm_uninstall_result.stderr.decode()}") + print(f"Error uninstalling deployment '{connector}': {helm_uninstall_result.stderr.decode()}") def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None @@ -130,6 +132,9 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): ) break + if result is None: + result = ActionResponse(status="OK", status_code=200) + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -178,7 +183,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): ) if err is None: - result = ActionResponse(status="OK", status_code=200) + result = ActionResponse(status="OK", status_code=200) return result else: @@ -244,10 +249,9 @@ def _perform_flink_install(self, dataset_id, connector_instance): def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None - release_name = "{}-{}-{}".format( - dataset_id[:8].lower().replace("_", "-"), - connector_instance.connector_id[:8].lower().replace("_", "-"), - str(time.time_ns()) + release_name = "{}-{}".format( + (dataset_id + "_" + connector_instance.connector_id).lower().replace("_", "-")[:54], + "".join(choice(ascii_lowercase) for i in range(6)) ) connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] @@ -362,10 +366,10 @@ def _get_registered_connectors(self, runtime): query = f""" SELECT cr.id, COUNT(ci.id) AS instance_count FROM connector_registry cr - LEFT JOIN connector_instances ci ON cr.id = ci.connector_id + LEFT JOIN connector_instances ci ON (cr.id = ci.connector_id AND ci.status = %s) WHERE cr.runtime = %s GROUP BY cr.id """ - params = (runtime,) + params = (DatasetStatusType.Live.name, runtime) rows = self.db_service.execute_select_all(sql=query, params=params) return rows \ No newline at end of file From f2870771d224332c676e2fefe194703900b633b3 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Mon, 30 Dec 2024 14:59:38 +0530 Subject: [PATCH 264/311] #OBS-I435: updated defaults --- api-service/src/configs/DatasetConfigDefault.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/DatasetConfigDefault.ts b/api-service/src/configs/DatasetConfigDefault.ts index 27bdeda0..5c1802dc 100644 --- a/api-service/src/configs/DatasetConfigDefault.ts +++ b/api-service/src/configs/DatasetConfigDefault.ts @@ -17,7 +17,7 @@ export const defaultDatasetConfig = { } }, "dedup_config": { - "drop_duplicates": true, + "drop_duplicates": false, "dedup_key": "id", "dedup_period": 604800, // 7 days }, From 724839f303036ee8f2e3e86f7e7e9e5ff6dc9308 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Mon, 30 Dec 2024 16:42:47 +0530 Subject: [PATCH 265/311] #I176: Integration of wrapper API to pull the datasources information from the table of datasources of postgres db instead of druid --- .../QueryWrapper/SqlQueryWrapper.ts | 47 +++++++++++++++++-- api-service/src/services/DatasourceService.ts | 19 ++++---- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts index dc1f0863..da374bd0 100644 --- a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts +++ b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts @@ -5,6 +5,8 @@ import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; import { druidHttpService } from "../../connections/druidConnection"; +import { getDatasourceList } from "../../services/DatasourceService"; +import { AxiosResponse } from "axios"; const apiId = "api.obsrv.data.sql-query"; const errorCode = "SQL_QUERY_FAILURE" @@ -25,10 +27,16 @@ export const sqlQuery = async (req: Request, res: Response) => { } as ErrorObject, req, res); } - const result = await druidHttpService.post(`${config.query_api.druid.sql_query_path}`, req.body, { - headers: { Authorization: authorization }, - }); - + const query = req.body.query as string; + let result: AxiosResponse; + if (isTableSchemaQuery(query)) { + const dataSources = await fetchDruidDataSources(); + result = createMockAxiosResponse(dataSources); + } else { + result = await druidHttpService.post(`${config.query_api.druid.sql_query_path}`, req.body, { + headers: { Authorization: authorization }, + }); + } logger.info({ messsge: "Successfully fetched data using sql query", apiId, resmsgid }) ResponseHandler.flatResponse(req, res, result) } catch (error: any) { @@ -37,4 +45,33 @@ export const sqlQuery = async (req: Request, res: Response) => { logger.error(error, apiId, code, resmsgid) ResponseHandler.errorResponse(errorMessage, req, res); } -} \ No newline at end of file +} + +const fetchDruidDataSources = async (): Promise<{ TABLE_NAME: string }[]> => { + try { + const dataSources = await getDatasourceList(); + return dataSources + .filter((ds: any) => ds.type === "druid") + .map((ds: any) => ({ TABLE_NAME: ds.dataValues.datasource_ref })); + } catch (error) { + logger.error({ message: "Failed to fetch Druid data sources", error }); + throw new Error("Failed to fetch Druid data sources"); + } +}; + +const isTableSchemaQuery = (sqlQuery?: string): boolean => { + return ( + sqlQuery?.trim() === + "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'druid'" + ); +}; + +const createMockAxiosResponse = (data: any): AxiosResponse => { + return { + data, + status: 200, + statusText: "OK", + headers: {}, + config: {}, + } as AxiosResponse; +}; diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index 812a5611..b5838900 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,16 +1,13 @@ import { Datasource } from "../models/Datasource"; -export const getDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await Datasource.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - - +export const getDatasourceList = async (datasetId?: string, raw: boolean = false) => { + const query: any = { raw }; + if (datasetId) { + query.where = { dataset_id: datasetId }; + } + + return Datasource.findAll(query); +}; From 03e21f736a292bb0a5ae0e699656cfaf52a24c20 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 30 Dec 2024 16:46:14 +0530 Subject: [PATCH 266/311] #OBS-I408: data schema fix for proper mappings and data type suggestions --- .../SchemaGenerateService/SchemaHandler.ts | 16 ++++++++++------ .../SchemaGenerateService/SchemaMapping.json | 8 ++++++-- .../SchemaGenerateService/SuggestionTemplate.ts | 12 +++++++++++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts index 45d76c9c..ac09027e 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts @@ -43,8 +43,11 @@ export class SchemaHandler { return _.set(schema, `${absolutePath}`, { ...schema[absolutePath], ...{ - type: resolution.value, - oneof: conflict.schema.values.map(key => ({ type: key })), + type: resolution.value || _.first(conflict.schema.values), + oneof: conflict.schema.values.map(key => { + const storeFormat = _.get(dataMappingPaths, key); + return { type: _.get(DataMappings, storeFormat) } + }), } }); } @@ -89,12 +92,13 @@ export class SchemaHandler { } private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { - const types = _.get(fieldData, type); - types && types.map((item: any) => { - const storeFormat = _.get(dataMappingPaths, item.type); + const types = _.get(fieldData, type) + const propType = _.get(fieldData, "type") + if(types){ + const storeFormat = _.get(dataMappingPaths, propType); _.set(schema, `${property}.arrival_format`, _.first(storeFormat.split("."))); _.set(schema, `${property}.data_type`, _.get(DataMappings, storeFormat)); - }) + } return; } diff --git a/api-service/src/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json index e9fe2a6f..f2461a05 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaMapping.json +++ b/api-service/src/services/SchemaGenerateService/SchemaMapping.json @@ -48,7 +48,7 @@ "datasource": "long" }, "float": { - "jsonSchema": "number", + "jsonSchema": "float", "datasource": "double" }, "long": { @@ -56,7 +56,7 @@ "datasource": "long" }, "double": { - "jsonSchema": "number", + "jsonSchema": "double", "datasource": "double" }, "bigdecimal": { @@ -66,6 +66,10 @@ "epoch":{ "jsonSchema": "integer", "datasource": "long" + }, + "number":{ + "jsonSchema": "double", + "datasource": "double" } } }, diff --git a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts index fcaba4a5..176ea74a 100644 --- a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts +++ b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts @@ -2,12 +2,22 @@ import _ from "lodash" import { Conflict, ConflictTypes, Suggestion, SuggestionsTemplate } from "../../types/SchemaModel" import constants from "./Constants.json" import { SchemaSuggestionTemplate } from "./Template" +import { dataMappingPaths } from "./SchemaHandler" +import DataMappings from "./SchemaMapping.json"; export class SuggestionTemplate { public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { return _.map(sample, (value) => { - const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) + let updatedConflicts: any = {} + if (value.schema.conflicts) { + updatedConflicts = _.mapKeys(value.schema.conflicts, (value, key) => { + const storeFormat = _.get(dataMappingPaths, key); + if (!storeFormat) return key + return _.get(DataMappings, storeFormat); + }) + } + const dataTypeSuggestions = this.getSchemaMessageTemplate({ ...value.schema, ...(!_.isEmpty(updatedConflicts) && { conflicts: updatedConflicts }) }) const requiredSuggestions = this.getRequiredMessageTemplate(value.required) const formatSuggestions = this.getPropertyFormatTemplate(value.formats) return { From 0b3b4a50036dde2908ef6ea73ed0124dc4e6017c Mon Sep 17 00:00:00 2001 From: Manjunath Date: Mon, 30 Dec 2024 17:00:13 +0530 Subject: [PATCH 267/311] #I176: Review comment fix - Case senstive fix --- .../src/controllers/QueryWrapper/SqlQueryWrapper.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts index da374bd0..470a479b 100644 --- a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts +++ b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts @@ -61,10 +61,14 @@ const fetchDruidDataSources = async (): Promise<{ TABLE_NAME: string }[]> => { const isTableSchemaQuery = (sqlQuery?: string): boolean => { return ( - sqlQuery?.trim() === - "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'druid'" + sqlQuery + ?.trim() + .replace(/\s+/g, " ") + .toUpperCase() === + "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'DRUID'" ); -}; + }; + const createMockAxiosResponse = (data: any): AxiosResponse => { return { From ac306e672ddff1d2b3624e4d78736e3a9820f9a4 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 31 Dec 2024 16:22:53 +0530 Subject: [PATCH 268/311] fix: #OBS-I441 added labels for cronjob --- .../helm-charts/spark-connector-cron/templates/cronjob.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index a80831fc..14c3d9b1 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -6,6 +6,7 @@ metadata: labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} app.kubernetes.io/dataset: {{ .Values.dataset_id }} app.kubernetes.io/connector: {{ .Values.connector_id }} + app.kubernetes.io/connector_instance: {{ .Values.instance_id }} {{- if .Values.commonAnnotations }} annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} @@ -23,6 +24,7 @@ spec: labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} app.kubernetes.io/dataset: {{ .Values.dataset_id }} app.kubernetes.io/connector: {{ .Values.connector_id }} + app.kubernetes.io/connector_instance: {{ .Values.instance_id }} spec: serviceAccountName: {{ .Values.serviceAccount.name }} restartPolicy: {{ .Values.restartPolicy }} From 02e858625ece5b0054625f3683094cefdddde7cf Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 31 Dec 2024 18:40:09 +0530 Subject: [PATCH 269/311] fix: #OBS-I406 storage type support read from envs --- api-service/src/configs/Config.ts | 3 ++- .../src/controllers/DatasetCopy/DatasetCopy.ts | 3 ++- .../src/controllers/DatasetCreate/DatasetCreate.ts | 3 ++- .../src/controllers/DatasetImport/DatasetImport.ts | 3 ++- .../DatasetStatusTransition.ts | 4 +++- .../src/controllers/DatasetUpdate/DatasetUpdate.ts | 3 ++- api-service/src/services/DatasetService.ts | 12 ++++++++++++ 7 files changed, 25 insertions(+), 6 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index cd74c111..6f8ab456 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -118,5 +118,6 @@ export const config = { "otel": { "enable": process.env.otel_enable || "false", "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" - } + }, + "storage_support": process.env.STORAGE_TYPE || 'lakehouse,druid' } diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 9015e9c7..7c53f538 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -4,7 +4,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import * as _ from "lodash"; import { schemaValidation } from "../../services/ValidationService"; import validationSchema from "./RequestValidationSchema.json"; -import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import { datasetService, getLiveDatasetConfigs, validateStorageSupport } from "../../services/DatasetService"; import { updateRecords } from "./DatasetCopyHelper"; import { obsrvError } from "../../types/ObsrvError"; @@ -40,6 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); + validateStorageSupport(dataset); const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); _.set(dataset, "updated_by", userID); diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index bb96dac2..d91da0f5 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -1,7 +1,7 @@ import _ from "lodash"; import { Request, Response } from "express"; import httpStatus from "http-status"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; @@ -28,6 +28,7 @@ const validateRequest = async (req: Request) => { throw obsrvError(datasetId, "DATASET_DUPLICATE_DENORM_KEY", "Duplicate denorm output fields found.", "BAD_REQUEST", 400, undefined, {duplicateKeys: duplicateDenormKeys}) } + validateStorageSupport(_.get(req, ["body", "request"])) } const datasetCreate = async (req: Request, res: Response) => { diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 2d0312b5..de326bb9 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import _ from "lodash"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { datasetImportValidation, migrateExportedDatasetV1 } from "./DatasetImportHelper"; import { obsrvError } from "../../types/ObsrvError"; @@ -21,6 +21,7 @@ const datasetImport = async (req: Request, res: Response) => { const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) + validateStorageSupport(updatedDataset); const dataset = await importDataset(updatedDataset, overwrite, userID); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 0197d39e..ff1502d7 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -1,7 +1,7 @@ import { Request, Response } from "express"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import StatusTransitionSchema from "./RequestValidationSchema.json"; import ReadyToPublishSchema from "./ReadyToPublishSchema.json" @@ -88,6 +88,7 @@ const deleteDataset = async (dataset: Record) => { const readyForPublish = async (dataset: Record, updated_by: any) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + validateStorageSupport(draftDataset); let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); @@ -136,6 +137,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record + validateStorageSupport(draftDataset); _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index e9fea051..c1ef0324 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -5,7 +5,7 @@ import Model from "sequelize/types/model"; import { DatasetStatus } from "../../types/DatasetModels"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { cipherService } from "../../services/CipherService"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; import { obsrvError } from "../../types/ObsrvError"; @@ -30,6 +30,7 @@ const validateRequest = async (req: Request) => { throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) } + validateStorageSupport(_.get(req, ["body", "request"])) } const validateDataset = (dataset: Record | null, req: Request) => { diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index e8ce9a35..59506b72 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -18,6 +18,7 @@ import { obsrvError } from "../types/ObsrvError"; import { druidHttpService } from "../connections/druidConnection"; import { tableGenerator } from "./TableGenerator"; import { deleteAlertByDataset, deleteMetricAliasByDataset } from "./managers"; +import { config } from "../configs/Config"; class DatasetService { @@ -411,4 +412,15 @@ export const getV1Connectors = async (datasetId: string) => { return modifiedV1Connectors; } +const storageSupport = _.split(config.storage_support, ",") +export const validateStorageSupport = (dataset: Record) => { + const { olap_store_enabled, lakehouse_enabled } = _.get(dataset, ["dataset_config", "indexing_config"]) || {} + if (olap_store_enabled && !_.includes(storageSupport, "druid")) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "olap_store" is not available. Please use one of the available storage types: ${storageSupport}`, "BAD_REQUEST", 400) + } + if (lakehouse_enabled && !_.includes(storageSupport, "lakehouse")) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "lakehouse" is not available. Please use one of the available storage types: ${storageSupport}`, "BAD_REQUEST", 400) + } +} + export const datasetService = new DatasetService(); \ No newline at end of file From 0f5197d4745e5f59ce936d3cf75e6194d24e2a6d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 2 Jan 2025 12:21:45 +0530 Subject: [PATCH 270/311] fix: #OBS-I406 storage type support envs --- api-service/src/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 6f8ab456..77c1caec 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -119,5 +119,5 @@ export const config = { "enable": process.env.otel_enable || "false", "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" }, - "storage_support": process.env.STORAGE_TYPE || 'lakehouse,druid' + "storage_support": process.env.storage_type || 'lakehouse,druid' } From d107ebdb9ad5e607b2f7f504f8c455b1f39898db Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 2 Jan 2025 14:56:06 +0530 Subject: [PATCH 271/311] fix: #OBS-I406 plural envs --- api-service/src/configs/Config.ts | 2 +- api-service/src/services/DatasetService.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 77c1caec..1b4ccb04 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -119,5 +119,5 @@ export const config = { "enable": process.env.otel_enable || "false", "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" }, - "storage_support": process.env.storage_type || 'lakehouse,druid' + "storage_types": process.env.storage_types || 'druid,datalake' } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 59506b72..186e43b5 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -412,14 +412,14 @@ export const getV1Connectors = async (datasetId: string) => { return modifiedV1Connectors; } -const storageSupport = _.split(config.storage_support, ",") +const storageTypes = _.split(config.storage_types, ",") export const validateStorageSupport = (dataset: Record) => { const { olap_store_enabled, lakehouse_enabled } = _.get(dataset, ["dataset_config", "indexing_config"]) || {} - if (olap_store_enabled && !_.includes(storageSupport, "druid")) { - throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "olap_store" is not available. Please use one of the available storage types: ${storageSupport}`, "BAD_REQUEST", 400) + if (olap_store_enabled && !_.includes(storageTypes, "druid")) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "olap_store" is not available. Please use one of the available storage types: ${storageTypes}`, "BAD_REQUEST", 400) } - if (lakehouse_enabled && !_.includes(storageSupport, "lakehouse")) { - throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "lakehouse" is not available. Please use one of the available storage types: ${storageSupport}`, "BAD_REQUEST", 400) + if (lakehouse_enabled && !_.includes(storageTypes, "datalake")) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "datalake" is not available. Please use one of the available storage types: ${storageTypes}`, "BAD_REQUEST", 400) } } From 30c0b28973fbd58a480c867dd794c2058f154612 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 3 Jan 2025 12:52:58 +0530 Subject: [PATCH 272/311] fix: #OBS-I406 storage type support envs as json --- api-service/src/configs/Config.ts | 2 +- api-service/src/services/DatasetService.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 1b4ccb04..a2aa79a1 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -119,5 +119,5 @@ export const config = { "enable": process.env.otel_enable || "false", "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" }, - "storage_types": process.env.storage_types || 'druid,datalake' + "storage_types": process.env.storage_types || '{"lake_house":true,"realtime_store":true}' } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 186e43b5..a5821892 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -412,14 +412,15 @@ export const getV1Connectors = async (datasetId: string) => { return modifiedV1Connectors; } -const storageTypes = _.split(config.storage_types, ",") +const storageTypes = JSON.parse(config.storage_types) export const validateStorageSupport = (dataset: Record) => { const { olap_store_enabled, lakehouse_enabled } = _.get(dataset, ["dataset_config", "indexing_config"]) || {} - if (olap_store_enabled && !_.includes(storageTypes, "druid")) { - throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "olap_store" is not available. Please use one of the available storage types: ${storageTypes}`, "BAD_REQUEST", 400) + const validStorageType = _.keys(storageTypes).filter(key => storageTypes[key] === true); + if (olap_store_enabled && !_.get(storageTypes, "realtime_store") === true) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "realtime_store" is not available. Please use one of the available storage types: ${validStorageType}`, "BAD_REQUEST", 400) } - if (lakehouse_enabled && !_.includes(storageTypes, "datalake")) { - throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "datalake" is not available. Please use one of the available storage types: ${storageTypes}`, "BAD_REQUEST", 400) + if (lakehouse_enabled && !_.get(storageTypes, "lake_house") == true) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "lake_house" is not available. Please use one of the available storage types: ${validStorageType}`, "BAD_REQUEST", 400) } } From ce5b774e1321cfe9ed93080c265eff2020ab2795 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Thu, 9 Jan 2025 15:20:43 +0530 Subject: [PATCH 273/311] fix #OBS-I479 ingestion batch structure fix --- .../DataIngestion/DataIngestionController.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 46f829e9..5044bb43 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -39,34 +39,34 @@ const dataIn = async (req: Request, res: Response) => { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); } - const { entry_topic, dataset_config, api_version } = dataset + const { entry_topic, dataset_config, extraction_config, api_version } = dataset const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic if (!entryTopic) { logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); } - await send(addMetadataToEvents(datasetId, requestBody), entryTopic) + await send(addMetadataToEvents(datasetId, requestBody, extraction_config), entryTopic) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); } -const addMetadataToEvents = (datasetId: string, payload: any) => { +const addMetadataToEvents = (datasetId: string, payload: any, extraction_config: any) => { const validData = _.get(payload, "data"); const now = Date.now(); const mid = _.get(payload, "params.msgid"); const source = { id: "api.data.in", version: config?.version, entry_source: "api" }; const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source }; if (Array.isArray(validData)) { - const payloadRef = validData.map((event: any) => { - const payload = { - event, + const extraction_key: string = _.get(extraction_config, "extraction_key", 'events'); + const dedup_key: string = _.get(extraction_config, "dedup_config.dedup_key", 'id'); + const payload: any = { "obsrv_meta": obsrvMeta, "dataset": datasetId, "msgid": mid - } + }; + payload[extraction_key] = validData; + payload[dedup_key] = mid return payload; - }) - return payloadRef; } else { return ({ From d6937f92efeb253939b6fdea2a99ab2a286b1891 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 9 Jan 2025 16:05:20 +0530 Subject: [PATCH 274/311] fix: #OBS-I354 dataset level alert metric caching issue fix --- .../src/command/alert_manager_command.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 3b6a82b9..5915a54d 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -5,7 +5,7 @@ from model.data_models import Action, ActionResponse, CommandPayload from service.db_service import DatabaseService from service.http_service import HttpService - +from copy import deepcopy class AlertManagerService(ICommand): @@ -95,17 +95,18 @@ def get_dataset_source_config(self, dataset_id: str) -> str: def get_modified_metric( self, service: str, metric: dict, payload: CommandPayload ) -> dict: + metric_data = deepcopy(metric) if service == "flink": substring = f"{payload.dataset_id}" modified_substring = substring.replace("-", "_") - modified_metric = metric["metric"].replace("dataset_id", modified_substring) - metric["metric"] = modified_metric - return metric + modified_metric = metric_data["metric"].replace("dataset_id", modified_substring) + metric_data["metric"] = modified_metric + return metric_data else: - metric["metric"] = metric["metric"].replace( + metric_data["metric"] = metric_data["metric"].replace( "dataset_id", payload.dataset_id ) - return metric + return metric_data def create_alert_metric( self, payload: CommandPayload, service: str, metric: dict, dataset_name: str From 863ea5337b403a6cf437c47d46aabf5dfcb6e8e6 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 9 Jan 2025 17:55:14 +0530 Subject: [PATCH 275/311] fix: #OBS-I354 ingestion spec expression fix for transformations --- api-service/src/configs/IngestionConfig.ts | 2 +- api-service/src/services/TableGenerator.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts index cfc4d1cf..1ac3810e 100644 --- a/api-service/src/configs/IngestionConfig.ts +++ b/api-service/src/configs/IngestionConfig.ts @@ -84,7 +84,7 @@ export const rawIngestionSpecDefaults = { "flattenSpec": [ { "type": "path", - "expr": "$.obsrv_meta.source.connector", + "expr": "$.obsrv_meta.source.entry_source", "name": "obsrv.meta.source.connector" }, { diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index 661a3919..a3e24d58 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -66,7 +66,7 @@ class BaseTableGenerator { } if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ - expr: "$." + `['${tf.field_key}']`, + expr: "$." + tf.field_key.split('.').map((fieldpart: string) => `['${fieldpart}']`).join('.'), name: tf.field_key, data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, From 2f6844150a3602279d8a43fdbb21256a44704f61 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 9 Jan 2025 17:59:27 +0530 Subject: [PATCH 276/311] fix: #OBS-I354 default field expr fix --- api-service/src/configs/IngestionConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts index 1ac3810e..52d71a85 100644 --- a/api-service/src/configs/IngestionConfig.ts +++ b/api-service/src/configs/IngestionConfig.ts @@ -89,7 +89,7 @@ export const rawIngestionSpecDefaults = { }, { "type": "path", - "expr": "$.obsrv_meta.source.connectorInstance", + "expr": "$.obsrv_meta.source.id", "name": "obsrv.meta.source.id" } ], From f3175e55e15a26a54193095377f0043b68c116aa Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 10 Jan 2025 16:40:29 +0530 Subject: [PATCH 277/311] fix: #OBS-I494: updated permissions for admin user --- api-service/src/middlewares/userPermissions.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index 400e218f..eda444c4 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -4,6 +4,7 @@ "api.datasets.list", "api.datasets.read", "api.datasets.export", + "api.dataset.health", "api.data.out", "api.data.exhaust", "api.alert.list", @@ -46,7 +47,6 @@ "api.datasets.update", "api.datasets.import", "api.datasets.copy", - "api.dataset.health", "api.datasets.dataschema" ], "data": [ @@ -116,7 +116,11 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "alert", + "metric", + "silence", + "notificationChannel" ], "operations_admin": [ "alert", From 55f48dfd3b7934f2e0f7fc937bd061f66122a5fa Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 13 Jan 2025 00:30:26 +0530 Subject: [PATCH 278/311] feat: #OBS-I321: Added logic to data_freshness,data_observability,data_volume and route and helper functions --- .../src/controllers/TableMetrics/Metrics.ts | 47 ++++++ .../TableMetrics/TableMetrics.json | 56 +++++++ .../controllers/TableMetrics/metricsHelper.ts | 128 ++++++++++++++++ .../src/controllers/TableMetrics/queries.ts | 142 ++++++++++++++++++ api-service/src/metrics/prometheus/helpers.ts | 2 +- 5 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 api-service/src/controllers/TableMetrics/Metrics.ts create mode 100644 api-service/src/controllers/TableMetrics/TableMetrics.json create mode 100644 api-service/src/controllers/TableMetrics/metricsHelper.ts create mode 100644 api-service/src/controllers/TableMetrics/queries.ts diff --git a/api-service/src/controllers/TableMetrics/Metrics.ts b/api-service/src/controllers/TableMetrics/Metrics.ts new file mode 100644 index 00000000..b86895eb --- /dev/null +++ b/api-service/src/controllers/TableMetrics/Metrics.ts @@ -0,0 +1,47 @@ +import { Request, Response } from "express"; +import * as _ from "lodash" +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import dayjs from 'dayjs'; +import { handleDataFreshness, handleDataObservability, handleDataVolume } from "./metricsHelper"; +import logger from "../../logger"; + +const apiId = "api.table.metrics"; +const tableMetrics = async (req: Request, res: Response) => { + const { dataset_id } = req.params; + const msgid = _.get(req, "body.params.msgid"); + const requestBody = req.body; + + const { category, volume_by_days }: any = req.body.request; + const defaultThreshold = 5 * 60 * 1000; // 5 minutes in milliseconds read from env + const dateFormat = 'YYYY-MM-DDTHH:mm:ssZ'; + const startDate = '2000-01-01T00:00:00+05:30'; + const endDate = dayjs().add(1, 'day').format(dateFormat); + const intervals = `${startDate}/${endDate}`; + + const results = []; + + try { + if (!category || category.includes("data_freshness")) { + const dataFreshnessResult = await handleDataFreshness(dataset_id, intervals, defaultThreshold); + results.push(dataFreshnessResult); + } + + if (!category || category.includes("data_observability")) { + const dataObservabilityResult = await handleDataObservability(dataset_id, intervals); + results.push(dataObservabilityResult); + } + + if (!category || category.includes("data_volume")) { + const dataVolumeResult = await handleDataVolume(dataset_id, volume_by_days, dateFormat); + results.push(dataVolumeResult); + } + logger.info({ apiId, msgid, requestBody, datasetId: dataset_id, message: "Metrics fetched successfully" }) + return ResponseHandler.successResponse(req, res, { status: 200, data: results }); + } catch (error: any) { + logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS" }); + return ResponseHandler.errorResponse({ message: "Error while fetching metrics", statusCode: 500, errCode: "FAILED", code: "FAILED_TO_FETCH_METRICS" }, req, res); + } + +} + +export default tableMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/TableMetrics/TableMetrics.json b/api-service/src/controllers/TableMetrics/TableMetrics.json new file mode 100644 index 00000000..656a3b73 --- /dev/null +++ b/api-service/src/controllers/TableMetrics/TableMetrics.json @@ -0,0 +1,56 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "api.table.metrics" + ] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "category": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "data_freshness", + "data_observability", + "data_volume", + "data_lineage", + "connectors" + ] + }, + "minItems": 1 + }, + "volume_by_days": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "category" + ] + } + }, + "required": ["id", "ver", "ts", "params", "request"] +} \ No newline at end of file diff --git a/api-service/src/controllers/TableMetrics/metricsHelper.ts b/api-service/src/controllers/TableMetrics/metricsHelper.ts new file mode 100644 index 00000000..e7e25e0b --- /dev/null +++ b/api-service/src/controllers/TableMetrics/metricsHelper.ts @@ -0,0 +1,128 @@ +import axios from "axios"; +import dayjs from "dayjs"; +import _ from "lodash"; +import { config } from "../../configs/Config"; +import { generateTimeseriesQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "./queries"; +const druidPort = _.get(config, "query_api.druid.port"); +const druidHost = _.get(config, "query_api.druid.host"); +const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; + +export const handleDataFreshness = async (dataset_id: string, intervals: string, defaultThreshold: number) => { + const queryPayload = processingTimeQuery(intervals, dataset_id); + const druidResponse = await axios.post(nativeQueryEndpoint, queryPayload?.query); + const avgProcessingTime = _.get(druidResponse, "data[0].average_processing_time", 0); + const freshnessStatus = avgProcessingTime < defaultThreshold ? "Healthy" : "Unhealthy"; + + return { + category: "data_freshness", + status: freshnessStatus, + components: [ + { + type: "average_time_difference_in_min", + threshold: defaultThreshold / 60000, // convert to minutes + value: avgProcessingTime / 60000, + status: freshnessStatus + }, + { + type: "freshness_query_time_in_min", + threshold: 10, + value: avgProcessingTime / 60000, // convert to minutes + status: freshnessStatus + } + ] + }; +}; + +export const handleDataObservability = async (dataset_id: string, intervals: string) => { + const totalEventsPayload = totalEventsQuery(intervals, dataset_id); + const totalFailedEventsPayload = totalFailedEventsQuery(intervals, dataset_id); + + const [totalEventsResponse, totalFailedEventsResponse] = await Promise.all([ + axios.post(nativeQueryEndpoint, totalEventsPayload), + axios.post(nativeQueryEndpoint, totalFailedEventsPayload) + ]); + + const totalEventsCount = _.get(totalEventsResponse, "data[0].result.total_events_count", 0); + const totalFailedEventsCount = _.get(totalFailedEventsResponse, "data[0].result.total_failed_events", 0); + let failurePercentage = 0; + if (totalEventsCount > 0) { + failurePercentage = (totalFailedEventsCount / totalEventsCount) * 100; + } + const observabilityStatus = failurePercentage > 5 ? "Unhealthy" : "Healthy"; + + return { + category: "data_observability", + status: observabilityStatus, + components: [ + { + type: "data_observability_health", + status: observabilityStatus + }, + { + type: "failure_percentage", + value: failurePercentage + }, + { + type: "threshold_percentage", + value: 5 + } + ] + }; +}; + +export const handleDataVolume = async (dataset_id: string, volume_by_days: number, dateFormat: string) => { + const currentHourIntervals = dayjs().subtract(1, 'hour').format(dateFormat) + '/' + dayjs().format(dateFormat); + const currentDayIntervals = dayjs().subtract(1, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); + const currentWeekIntervals = dayjs().subtract(1, 'week').startOf('week').format(dateFormat) + '/' + dayjs().endOf('week').format(dateFormat); + const previousHourIntervals = dayjs().subtract(2, 'hour').format(dateFormat) + '/' + dayjs().subtract(1, 'hour').format(dateFormat); + const previousDayIntervals = dayjs().subtract(2, 'day').startOf('day').format(dateFormat) + '/' + dayjs().subtract(1, 'day').endOf('day').format(dateFormat); + const previousWeekIntervals = dayjs().subtract(2, 'week').startOf('week').format(dateFormat) + '/' + dayjs().subtract(1, 'week').endOf('week').format(dateFormat); + const nDaysIntervals = dayjs().subtract(volume_by_days, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); + + const currentHourPayload = generateTimeseriesQuery(currentHourIntervals, dataset_id); + const currentDayPayload = generateTimeseriesQuery(currentDayIntervals, dataset_id); + const currentWeekPayload = generateTimeseriesQuery(currentWeekIntervals, dataset_id); + const previousHourPayload = generateTimeseriesQuery(previousHourIntervals, dataset_id); + const previousDayPayload = generateTimeseriesQuery(previousDayIntervals, dataset_id); + const previousWeekPayload = generateTimeseriesQuery(previousWeekIntervals, dataset_id); + const nDaysPayload = generateTimeseriesQuery(nDaysIntervals, dataset_id); + + const [ + currentHourResponse, currentDayResponse, currentWeekResponse, + previousHourResponse, previousDayResponse, previousWeekResponse, + nDaysResponse + ] = await Promise.all([ + axios.post(nativeQueryEndpoint, currentHourPayload), + axios.post(nativeQueryEndpoint, currentDayPayload), + axios.post(nativeQueryEndpoint, currentWeekPayload), + axios.post(nativeQueryEndpoint, previousHourPayload), + axios.post(nativeQueryEndpoint, previousDayPayload), + axios.post(nativeQueryEndpoint, previousWeekPayload), + axios.post(nativeQueryEndpoint, nDaysPayload) + ]); + + const currentHourCount = _.get(currentHourResponse, "data[0].result.count", 0); + const currentDayCount = _.get(currentDayResponse, "data[0].result.count", 0); + const currentWeekCount = _.get(currentWeekResponse, "data[0].result.count", 0); + const previousHourCount = _.get(previousHourResponse, "data[0].result.count", 0); + const previousDayCount = _.get(previousDayResponse, "data[0].result.count", 0); + const previousWeekCount = _.get(previousWeekResponse, "data[0].result.count", 0); + const nDaysCount = _.get(nDaysResponse, "data[0].result.count", 0); + + const volumePercentageByHour = previousHourCount > 0 ? ((currentHourCount - previousHourCount) / previousHourCount) * 100 : 0; + const volumePercentageByDay = previousDayCount > 0 ? ((currentDayCount - previousDayCount) / previousDayCount) * 100 : 0; + const volumePercentageByWeek = previousWeekCount > 0 ? ((currentWeekCount - previousWeekCount) / previousWeekCount) * 100 : 0; + + return { + category: "data_volume", + components: [ + { type: "events_per_hour", value: currentHourCount }, + { type: "events_per_day", value: currentDayCount }, + { type: "events_per_n_day", value: nDaysCount }, + { type: "volume_percentage_by_hour", value: volumePercentageByHour }, + { type: "volume_percentage_by_day", value: volumePercentageByDay }, + { type: "volume_percentage_by_week", value: volumePercentageByWeek }, + { type: "growth_rate_percentage", value: volumePercentageByHour } // Assuming growth rate is same as volume percentage by hour + ] + }; +}; \ No newline at end of file diff --git a/api-service/src/controllers/TableMetrics/queries.ts b/api-service/src/controllers/TableMetrics/queries.ts new file mode 100644 index 00000000..8b2be352 --- /dev/null +++ b/api-service/src/controllers/TableMetrics/queries.ts @@ -0,0 +1,142 @@ +export const processingTimeQuery = (intervals: string, dataset: string) => ({ + query: { + queryType: "groupBy", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "Asia/Kolkata" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "processing_time", fieldName: "total_processing_time" }, + { type: "longSum", name: "count", fieldName: "count" } + ], + postAggregations: [ + { + type: "expression", + name: "average_processing_time", + expression: "case_searched((count > 0),(processing_time/count),0)" + } + ] + } + }); + + export const totalEventsQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "longSum", + name: "total_events_count", + fieldName: "count" + } + ], + context: { + queryId: "02dd33a4-5f3c-4b7e-8d21-56769799d413", + sqlOuterLimit: 1001, + sqlQueryId: "02dd33a4-5f3c-4b7e-8d21-56769799d413", + useNativeQueryExplain: true + } + }); + + export const totalFailedEventsQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "total_failed_events", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "validator" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "total_failed_events" + } + ], + context: { + queryId: "acf3990b-1c9e-42a6-9960-eb85a115ca65", + sqlOuterLimit: 1001, + sqlQueryId: "acf3990b-1c9e-42a6-9960-eb85a115ca65", + useNativeQueryExplain: true + } + }); + + export const generateTimeseriesQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "Asia/Kolkata" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] + }); \ No newline at end of file diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 05051461..febb5d59 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -58,7 +58,7 @@ const getMetricLabels = (req: any, res: Response) => { const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null + const dataset_id = _.get(req, ["body", "request", "datasetId"]) || _.get(req, ["params", "datasetId"]) || null const duration = getDuration(startTime); const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } From 6986230e7bb0e3ac7764fe54db17b9a65d66dd66 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 13 Jan 2025 00:32:01 +0530 Subject: [PATCH 279/311] feat: #OBS-I321: Added dayjs package --- api-service/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/package.json b/api-service/package.json index bf0a08fd..e80d65f7 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -42,6 +42,7 @@ "busboy": "^1.6.0", "compression": "^1.7.4", "dateformat": "2.0.0", + "dayjs": "^1.11.13", "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", From 0098317f89cd98c12aa61007757a6100e4106630 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 13 Jan 2025 19:17:10 +0530 Subject: [PATCH 280/311] feat: #OBS-I321: Updated queries and request validation schema --- .../TableMetrics/TableMetrics.json | 3 +- .../src/controllers/TableMetrics/queries.ts | 499 +++++++++++++----- 2 files changed, 378 insertions(+), 124 deletions(-) diff --git a/api-service/src/controllers/TableMetrics/TableMetrics.json b/api-service/src/controllers/TableMetrics/TableMetrics.json index 656a3b73..79923de8 100644 --- a/api-service/src/controllers/TableMetrics/TableMetrics.json +++ b/api-service/src/controllers/TableMetrics/TableMetrics.json @@ -37,7 +37,8 @@ "data_observability", "data_volume", "data_lineage", - "connectors" + "connectors", + "data_quality" ] }, "minItems": 1 diff --git a/api-service/src/controllers/TableMetrics/queries.ts b/api-service/src/controllers/TableMetrics/queries.ts index 8b2be352..41b72cdc 100644 --- a/api-service/src/controllers/TableMetrics/queries.ts +++ b/api-service/src/controllers/TableMetrics/queries.ts @@ -1,126 +1,8 @@ -export const processingTimeQuery = (intervals: string, dataset: string) => ({ - query: { - queryType: "groupBy", - dataSource: "system-events", - intervals: intervals, - granularity: { - type: "all", - timeZone: "Asia/Kolkata" - }, - filter: { - type: "and", - fields: [ - { type: "selector", dimension: "ctx_module", value: "processing" }, - { type: "selector", dimension: "ctx_dataset", value: dataset }, - { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, - { type: "selector", dimension: "error_code", value: null } - ] - }, - aggregations: [ - { type: "longSum", name: "processing_time", fieldName: "total_processing_time" }, - { type: "longSum", name: "count", fieldName: "count" } - ], - postAggregations: [ - { - type: "expression", - name: "average_processing_time", - expression: "case_searched((count > 0),(processing_time/count),0)" - } - ] - } - }); - - export const totalEventsQuery = (intervals: string, dataset: string) => ({ - queryType: "timeseries", - dataSource: { - type: "table", - name: "system-events" - }, - intervals: { - type: "intervals", - intervals: [intervals] - }, - filter: { - type: "equals", - column: "ctx_dataset", - matchValueType: "STRING", - matchValue: dataset - }, - granularity: { - type: "all" - }, - aggregations: [ - { - type: "longSum", - name: "total_events_count", - fieldName: "count" - } - ], - context: { - queryId: "02dd33a4-5f3c-4b7e-8d21-56769799d413", - sqlOuterLimit: 1001, - sqlQueryId: "02dd33a4-5f3c-4b7e-8d21-56769799d413", - useNativeQueryExplain: true - } - }); - - export const totalFailedEventsQuery = (intervals: string, dataset: string) => ({ - queryType: "timeseries", - dataSource: { - type: "table", - name: "system-events" - }, - intervals: { - type: "intervals", - intervals: [intervals] - }, - filter: { - type: "equals", - column: "ctx_dataset", - matchValueType: "STRING", - matchValue: dataset - }, - granularity: { - type: "all" - }, - aggregations: [ - { - type: "filtered", - aggregator: { - type: "longSum", - name: "total_failed_events", - fieldName: "count" - }, - filter: { - type: "and", - fields: [ - { - type: "equals", - column: "ctx_pdata_pid", - matchValueType: "STRING", - matchValue: "validator" - }, - { - type: "equals", - column: "error_pdata_status", - matchValueType: "STRING", - matchValue: "failed" - } - ] - }, - name: "total_failed_events" - } - ], - context: { - queryId: "acf3990b-1c9e-42a6-9960-eb85a115ca65", - sqlOuterLimit: 1001, - sqlQueryId: "acf3990b-1c9e-42a6-9960-eb85a115ca65", - useNativeQueryExplain: true - } - }); +import dayjs from "dayjs"; - export const generateTimeseriesQuery = (intervals: string, dataset: string) => ({ - queryType: "timeseries", +export const processingTimeQuery = (intervals: string, dataset: string) => ({ + query: { + queryType: "groupBy", dataSource: "system-events", intervals: intervals, granularity: { @@ -137,6 +19,377 @@ export const processingTimeQuery = (intervals: string, dataset: string) => ({ ] }, aggregations: [ + { type: "longSum", name: "processing_time", fieldName: "total_processing_time" }, { type: "longSum", name: "count", fieldName: "count" } + ], + postAggregations: [ + { + type: "expression", + name: "average_processing_time", + expression: "case_searched((count > 0),(processing_time/count),0)" + } + ] + } +}); + +export const totalEventsQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "longSum", + name: "total_events_count", + fieldName: "count" + } + ] +}); + +export const totalFailedEventsQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "total_failed_events", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "validator" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "total_failed_events" + } + ] +}); + +export const generateTimeseriesQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "Asia/Kolkata" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] +}); + +export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "Asia/Kolkata" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] +}); + +export const dataLineageSuccessQuery = (intervals: string, dataset: string, column: string, value: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: column, + matchValueType: "STRING", + matchValue: value + }, + { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + } ] - }); \ No newline at end of file + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "longSum", + name: "count", + fieldName: "count" + } + ] +}); + +export const generateTransformationFailedQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_id", + matchValueType: "STRING", + matchValue: "TransformerJob" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "count" + } + ] +}); + +export const generateDedupFailedQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "dedup" + }, + { + type: "equals", + column: "error_type", + matchValueType: "STRING", + matchValue: "DedupFailed" + } + ] + }, + name: "count" + } + ] +}); + +export const generateDenormFailedQuery = (intervals: string, dataset: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "denorm" + }, + { + type: "equals", + column: "error_type", + matchValueType: "STRING", + matchValue: "DenormDataNotFound" + } + ] + }, + name: "count" + } + ] +}); + +export const generateConnectorQuery = (intervals: string, dataset: string) => ({ + queryType: "topN", + dataSource: { + type: "table", + name: "system-events" + }, + dimension: { + type: "default", + dimension: "ctx_source_connector", + outputName: "name", + outputType: "STRING" + }, + metric: { + type: "dimension", + ordering: { + type: "lexicographic" + } + }, + threshold: 1001, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset + }, + granularity: { + type: "all" + }, + aggregations: [ + { + type: "longSum", + name: "count", + fieldName: "count" + } + ] +}); + +export const generateTotalQueryCallsQuery = (time_period: string) => ({ + end: dayjs().unix(), + query: `sum(sum_over_time(node_total_api_calls{entity="data-out"}[${time_period}]))`, + step: `${time_period}`, + start: dayjs().subtract(1, 'day').unix() +}); + +export const generateDatasetQueryCallsQuery = (dataset: string, time_period: string) => ({ + end: dayjs().unix(), + step: `${time_period}`, + query: `sum(sum_over_time(node_total_api_calls{dataset_id="${dataset}",entity="data-out"}[${time_period}]))`, + start: dayjs().subtract(1, 'day').unix(), +}); \ No newline at end of file From a6ef18b9bef768c08ac227b7ba4baa636d609660 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 13 Jan 2025 19:17:29 +0530 Subject: [PATCH 281/311] feat: #OBS-I321: Added sections in controller --- .../src/controllers/TableMetrics/Metrics.ts | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/api-service/src/controllers/TableMetrics/Metrics.ts b/api-service/src/controllers/TableMetrics/Metrics.ts index b86895eb..f1ed75d2 100644 --- a/api-service/src/controllers/TableMetrics/Metrics.ts +++ b/api-service/src/controllers/TableMetrics/Metrics.ts @@ -2,8 +2,11 @@ import { Request, Response } from "express"; import * as _ from "lodash" import { ResponseHandler } from "../../helpers/ResponseHandler"; import dayjs from 'dayjs'; -import { handleDataFreshness, handleDataObservability, handleDataVolume } from "./metricsHelper"; +import { handleConnectors, handleDataFreshness, handleDataLineage, handleDataObservability, handleDataQuality, handleDataVolume } from "./metricsHelper"; import logger from "../../logger"; +import { schemaValidation } from "../../services/ValidationService"; +import validationSchema from "./TableMetrics.json"; +import { config } from "../../configs/Config"; const apiId = "api.table.metrics"; const tableMetrics = async (req: Request, res: Response) => { @@ -12,13 +15,17 @@ const tableMetrics = async (req: Request, res: Response) => { const requestBody = req.body; const { category, volume_by_days }: any = req.body.request; - const defaultThreshold = 5 * 60 * 1000; // 5 minutes in milliseconds read from env - const dateFormat = 'YYYY-MM-DDTHH:mm:ssZ'; + const defaultThreshold = (typeof config?.default_freshness_threshold === 'number' ? config.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds + const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; const startDate = '2000-01-01T00:00:00+05:30'; const endDate = dayjs().add(1, 'day').format(dateFormat); const intervals = `${startDate}/${endDate}`; - + const isValidSchema = schemaValidation(requestBody, validationSchema); const results = []; + if (!isValidSchema?.isValid) { + logger.error({ apiId, datasetId:dataset_id, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); + } try { if (!category || category.includes("data_freshness")) { @@ -35,10 +42,28 @@ const tableMetrics = async (req: Request, res: Response) => { const dataVolumeResult = await handleDataVolume(dataset_id, volume_by_days, dateFormat); results.push(dataVolumeResult); } + + if (!category || category.includes("data_lineage")) { + const dataLineageResult = await handleDataLineage(dataset_id, intervals); + results.push(dataLineageResult); + } + + if (!category || category.includes("connectors")) { + const connectorsResult = await handleConnectors(dataset_id, intervals); + results.push(connectorsResult); + } + + if (!category || category.includes("data_quality")) { + const connectorsResult = await handleDataQuality(dataset_id, intervals); + results.push(connectorsResult); + } + logger.info({ apiId, msgid, requestBody, datasetId: dataset_id, message: "Metrics fetched successfully" }) return ResponseHandler.successResponse(req, res, { status: 200, data: results }); - } catch (error: any) { - logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS" }); + + } + catch (error: any) { + logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS", error }); return ResponseHandler.errorResponse({ message: "Error while fetching metrics", statusCode: 500, errCode: "FAILED", code: "FAILED_TO_FETCH_METRICS" }, req, res); } From ac6d870a046b2e9fe5511e81d84612c541bcd2b6 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 13 Jan 2025 19:17:46 +0530 Subject: [PATCH 282/311] feat: #OBS-I321: Added logics to fetch data --- .../controllers/TableMetrics/metricsHelper.ts | 150 +++++++++++++++--- 1 file changed, 130 insertions(+), 20 deletions(-) diff --git a/api-service/src/controllers/TableMetrics/metricsHelper.ts b/api-service/src/controllers/TableMetrics/metricsHelper.ts index e7e25e0b..4413873a 100644 --- a/api-service/src/controllers/TableMetrics/metricsHelper.ts +++ b/api-service/src/controllers/TableMetrics/metricsHelper.ts @@ -2,10 +2,11 @@ import axios from "axios"; import dayjs from "dayjs"; import _ from "lodash"; import { config } from "../../configs/Config"; -import { generateTimeseriesQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "./queries"; +import { dataLineageSuccessQuery, generateConnectorQuery, generateDatasetQueryCallsQuery, generateDedupFailedQuery, generateDenormFailedQuery, generateTimeseriesQuery, generateTimeseriesQueryEventsPerHour, generateTotalQueryCallsQuery, generateTransformationFailedQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "./queries"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +const prometheusEndpoint = `${config.query_api.prometheus.url}/api/v1/query_range`; export const handleDataFreshness = async (dataset_id: string, intervals: string, defaultThreshold: number) => { const queryPayload = processingTimeQuery(intervals, dataset_id); @@ -20,7 +21,7 @@ export const handleDataFreshness = async (dataset_id: string, intervals: string, { type: "average_time_difference_in_min", threshold: defaultThreshold / 60000, // convert to minutes - value: avgProcessingTime / 60000, + value: avgProcessingTime / 60000, status: freshnessStatus }, { @@ -36,14 +37,28 @@ export const handleDataFreshness = async (dataset_id: string, intervals: string, export const handleDataObservability = async (dataset_id: string, intervals: string) => { const totalEventsPayload = totalEventsQuery(intervals, dataset_id); const totalFailedEventsPayload = totalFailedEventsQuery(intervals, dataset_id); + const totalQueryCalls = generateTotalQueryCallsQuery(config?.data_out_query_time_period); + const totalQueryCallsAtDatasetLevel = generateDatasetQueryCallsQuery(dataset_id, config?.data_out_query_time_period); - const [totalEventsResponse, totalFailedEventsResponse] = await Promise.all([ + const [totalEventsResponse, totalFailedEventsResponse, totalApiCallsResponse, totalCallsAtDatasetLevelResponse] = await Promise.all([ axios.post(nativeQueryEndpoint, totalEventsPayload), - axios.post(nativeQueryEndpoint, totalFailedEventsPayload) + axios.post(nativeQueryEndpoint, totalFailedEventsPayload), + axios.request({ url: prometheusEndpoint, method: "GET", params: totalQueryCalls }), + axios.request({ url: prometheusEndpoint, method: "GET", params: totalQueryCallsAtDatasetLevel }) ]); - const totalEventsCount = _.get(totalEventsResponse, "data[0].result.total_events_count", 0); - const totalFailedEventsCount = _.get(totalFailedEventsResponse, "data[0].result.total_failed_events", 0); + const totalApiCalls = _.map(_.get(totalApiCallsResponse, 'data.data.result'), payload => { + return _.floor(_.get(payload, 'values[0][1]'), 3) || 0 + }) + + const totalApiCallsAtDatasetLevel = _.map(_.get(totalCallsAtDatasetLevelResponse, 'data.data.result'), payload => { + return _.floor(_.get(payload, 'values[0][1]'), 3) || 0 + }) + + const importanceScore = (totalApiCallsAtDatasetLevel[0] / totalApiCalls[0]) * 100; + + const totalEventsCount = _.get(totalEventsResponse, "data[0].result.total_events_count") || 0; + const totalFailedEventsCount = _.get(totalFailedEventsResponse, "data[0].result.total_failed_events") || 0; let failurePercentage = 0; if (totalEventsCount > 0) { failurePercentage = (totalFailedEventsCount / totalEventsCount) * 100; @@ -65,28 +80,31 @@ export const handleDataObservability = async (dataset_id: string, intervals: str { type: "threshold_percentage", value: 5 + }, + { + type: "importance_score", + value: importanceScore } ] }; }; export const handleDataVolume = async (dataset_id: string, volume_by_days: number, dateFormat: string) => { - const currentHourIntervals = dayjs().subtract(1, 'hour').format(dateFormat) + '/' + dayjs().format(dateFormat); + const currentHourIntervals = dayjs().subtract(1, "hour").startOf("hour").toISOString() + "/" + dayjs().startOf("hour").toISOString(); const currentDayIntervals = dayjs().subtract(1, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); const currentWeekIntervals = dayjs().subtract(1, 'week').startOf('week').format(dateFormat) + '/' + dayjs().endOf('week').format(dateFormat); - const previousHourIntervals = dayjs().subtract(2, 'hour').format(dateFormat) + '/' + dayjs().subtract(1, 'hour').format(dateFormat); + const previousHourIntervals = dayjs().subtract(2, "hour").startOf("hour").toISOString() + '/' + dayjs().startOf("hour").toISOString(); const previousDayIntervals = dayjs().subtract(2, 'day').startOf('day').format(dateFormat) + '/' + dayjs().subtract(1, 'day').endOf('day').format(dateFormat); const previousWeekIntervals = dayjs().subtract(2, 'week').startOf('week').format(dateFormat) + '/' + dayjs().subtract(1, 'week').endOf('week').format(dateFormat); const nDaysIntervals = dayjs().subtract(volume_by_days, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); - const currentHourPayload = generateTimeseriesQuery(currentHourIntervals, dataset_id); + const currentHourPayload = generateTimeseriesQueryEventsPerHour(currentHourIntervals, dataset_id); const currentDayPayload = generateTimeseriesQuery(currentDayIntervals, dataset_id); const currentWeekPayload = generateTimeseriesQuery(currentWeekIntervals, dataset_id); - const previousHourPayload = generateTimeseriesQuery(previousHourIntervals, dataset_id); + const previousHourPayload = generateTimeseriesQueryEventsPerHour(previousHourIntervals, dataset_id); const previousDayPayload = generateTimeseriesQuery(previousDayIntervals, dataset_id); const previousWeekPayload = generateTimeseriesQuery(previousWeekIntervals, dataset_id); const nDaysPayload = generateTimeseriesQuery(nDaysIntervals, dataset_id); - const [ currentHourResponse, currentDayResponse, currentWeekResponse, previousHourResponse, previousDayResponse, previousWeekResponse, @@ -100,14 +118,13 @@ export const handleDataVolume = async (dataset_id: string, volume_by_days: numbe axios.post(nativeQueryEndpoint, previousWeekPayload), axios.post(nativeQueryEndpoint, nDaysPayload) ]); - - const currentHourCount = _.get(currentHourResponse, "data[0].result.count", 0); - const currentDayCount = _.get(currentDayResponse, "data[0].result.count", 0); - const currentWeekCount = _.get(currentWeekResponse, "data[0].result.count", 0); - const previousHourCount = _.get(previousHourResponse, "data[0].result.count", 0); - const previousDayCount = _.get(previousDayResponse, "data[0].result.count", 0); - const previousWeekCount = _.get(previousWeekResponse, "data[0].result.count", 0); - const nDaysCount = _.get(nDaysResponse, "data[0].result.count", 0); + const currentHourCount = _.get(currentHourResponse, "data[0].result.count") || 0; + const currentDayCount = _.get(currentDayResponse, "data[0].result.count") || 0; + const currentWeekCount = _.get(currentWeekResponse, "data[0].result.count") || 0; + const previousHourCount = _.get(previousHourResponse, "data[0].result.count") || 0; + const previousDayCount = _.get(previousDayResponse, "data[0].result.count") || 0; + const previousWeekCount = _.get(previousWeekResponse, "data[0].result.count") || 0; + const nDaysCount = _.get(nDaysResponse, "data[0].result.count") || 0; const volumePercentageByHour = previousHourCount > 0 ? ((currentHourCount - previousHourCount) / previousHourCount) * 100 : 0; const volumePercentageByDay = previousDayCount > 0 ? ((currentDayCount - previousDayCount) / previousDayCount) * 100 : 0; @@ -125,4 +142,97 @@ export const handleDataVolume = async (dataset_id: string, volume_by_days: numbe { type: "growth_rate_percentage", value: volumePercentageByHour } // Assuming growth rate is same as volume percentage by hour ] }; -}; \ No newline at end of file +}; + +export const handleDataLineage = async (dataset_id: string, intervals: string) => { + const transformationSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "transformer_status", "success"); + const dedupSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "dedup_status", "success"); + const denormSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "denorm_status", "success"); + const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); + const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); + const transformationFailedPayload = generateTransformationFailedQuery(intervals, dataset_id); + const dedupFailedPayload = generateDedupFailedQuery(intervals, dataset_id); + const denormFailedPayload = generateDenormFailedQuery(intervals, dataset_id); + + const [ + transformationSuccessResponse, dedupSuccessResponse, denormSuccessResponse, + totalValidationResponse, totalValidationFailedResponse, transformationFailedResponse, dedupFailedResponse, denormFailedResponse + ] = await Promise.all([ + axios.post(nativeQueryEndpoint, transformationSuccessPayload), + axios.post(nativeQueryEndpoint, dedupSuccessPayload), + axios.post(nativeQueryEndpoint, denormSuccessPayload), + axios.post(nativeQueryEndpoint, totalValidationPayload), + axios.post(nativeQueryEndpoint, totalValidationFailedPayload), + axios.post(nativeQueryEndpoint, transformationFailedPayload), + axios.post(nativeQueryEndpoint, dedupFailedPayload), + axios.post(nativeQueryEndpoint, denormFailedPayload) + ]); + + // success at each level + const transformationSuccessCount = _.get(transformationSuccessResponse, "data[0].result.count") || 0; + const dedupSuccessCount = _.get(dedupSuccessResponse, "data[0].result.count") || 0; + const denormSuccessCount = _.get(denormSuccessResponse, "data[0].result.count") || 0; + const totalValidationCount = _.get(totalValidationResponse, "data[0].result.count") || 0; + const totalValidationFailedCount = _.get(totalValidationFailedResponse, "data[0].result.count") || 0; + const storageSuccessCount = totalValidationCount - totalValidationFailedCount; + + // failure at each level + const transformationFailedCount = _.get(transformationFailedResponse, "data[0].result.count") || 0; + const dedupFailedCount = _.get(dedupFailedResponse, "data[0].result.count") || 0; + const denormFailedCount = _.get(denormFailedResponse, "data[0].result.count") || 0; + return { + category: "data_lineage", + components: [ + { type: "transformation_success", value: transformationSuccessCount }, + { type: "dedup_success", value: dedupSuccessCount }, + { type: "denormalization_success", value: denormSuccessCount }, + { type: "total_success", value: storageSuccessCount }, + { type: "total_failed", value: totalValidationFailedCount }, + { type: "transformation_failed", value: transformationFailedCount }, + { type: "dedup_failed", value: dedupFailedCount }, + { type: "denorm_failed", value: denormFailedCount } + ] + }; +}; + + +export const handleConnectors = async (dataset_id: string, intervals: string) => { + const connectorQueryPayload = generateConnectorQuery(intervals, dataset_id); + const connectorResponse = await axios.post(nativeQueryEndpoint, connectorQueryPayload); + const connectorsData = _.get(connectorResponse, "data[0].result", []); + const result = { + category: "connectors", + components: connectorsData.map((item: any) => ({ + id: item.name || "failed", + type: item.name === null ? "failed" : "success", + value: item.count + })) + }; + + return { + category: "connectors", + components: result + }; +}; + +export const handleDataQuality = async (dataset_id: string, intervals: string) => { + const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); + const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); + const [totalValidationResponse, totalValidationFailedResponse, + ] = await Promise.all([ + axios.post(nativeQueryEndpoint, totalValidationPayload), + axios.post(nativeQueryEndpoint, totalValidationFailedPayload), + ]); + const totalValidationCount = _.get(totalValidationResponse, "data[0].result.count") || 0; + const totalValidationFailedCount = _.get(totalValidationFailedResponse, "data[0].result.count") || 0; + const storageSuccessCount = totalValidationCount - totalValidationFailedCount; + + return { + category: "data_quality", + "components": [ + { type: "incidents_failed", value: totalValidationFailedCount }, + { type: "incidents_success", value: storageSuccessCount }, + { type: "total_incidents", value: totalValidationCount } + ] + }; +} \ No newline at end of file From 5f237953101fd6ce8116a4bac242422463747854 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 14 Jan 2025 16:14:30 +0530 Subject: [PATCH 283/311] feat: #OBS-I321: Added route --- api-service/src/routes/Router.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1e39d942..fb94237e 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -32,6 +32,7 @@ import telemetryActions from "../telemetry/telemetryActions"; import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; +import tableMetrics from "../controllers/TableMetrics/Metrics"; export const router = express.Router(); @@ -63,4 +64,4 @@ router.post("/connector/register", setDataToRequestObject("api.connector.registe //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) - +router.post("/table/metrics/:dataset_id",setDataToRequestObject("api.table.metrics"), tableMetrics); From 59dbce75ce3a2f1473caa693c81a1e7511e78363 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 14 Jan 2025 16:17:54 +0530 Subject: [PATCH 284/311] feat: #OBS-I321: Added config --- api-service/src/configs/Config.ts | 6 +++++- api-service/src/controllers/TableMetrics/Metrics.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index a2aa79a1..26a64a58 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -119,5 +119,9 @@ export const config = { "enable": process.env.otel_enable || "false", "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" }, - "storage_types": process.env.storage_types || '{"lake_house":true,"realtime_store":true}' + "storage_types": process.env.storage_types || '{"lake_house":true,"realtime_store":true}', + "data_observability": { + "default_freshness_threshold": process.env.default_freshness_threshold || 5, // in minutes + "data_out_query_time_period": process.env.data_out_query_time_period || "1d", + } } diff --git a/api-service/src/controllers/TableMetrics/Metrics.ts b/api-service/src/controllers/TableMetrics/Metrics.ts index f1ed75d2..513414c0 100644 --- a/api-service/src/controllers/TableMetrics/Metrics.ts +++ b/api-service/src/controllers/TableMetrics/Metrics.ts @@ -15,7 +15,7 @@ const tableMetrics = async (req: Request, res: Response) => { const requestBody = req.body; const { category, volume_by_days }: any = req.body.request; - const defaultThreshold = (typeof config?.default_freshness_threshold === 'number' ? config.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds + const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; const startDate = '2000-01-01T00:00:00+05:30'; const endDate = dayjs().add(1, 'day').format(dateFormat); From be0b49c07fceb0ee7b5434cc6737bdd0d4a8802c Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 14 Jan 2025 16:18:08 +0530 Subject: [PATCH 285/311] feat: #OBS-I321: Added config --- api-service/src/controllers/TableMetrics/metricsHelper.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/TableMetrics/metricsHelper.ts b/api-service/src/controllers/TableMetrics/metricsHelper.ts index 4413873a..58e8d386 100644 --- a/api-service/src/controllers/TableMetrics/metricsHelper.ts +++ b/api-service/src/controllers/TableMetrics/metricsHelper.ts @@ -37,8 +37,8 @@ export const handleDataFreshness = async (dataset_id: string, intervals: string, export const handleDataObservability = async (dataset_id: string, intervals: string) => { const totalEventsPayload = totalEventsQuery(intervals, dataset_id); const totalFailedEventsPayload = totalFailedEventsQuery(intervals, dataset_id); - const totalQueryCalls = generateTotalQueryCallsQuery(config?.data_out_query_time_period); - const totalQueryCallsAtDatasetLevel = generateDatasetQueryCallsQuery(dataset_id, config?.data_out_query_time_period); + const totalQueryCalls = generateTotalQueryCallsQuery(config?.data_observability?.data_out_query_time_period); + const totalQueryCallsAtDatasetLevel = generateDatasetQueryCallsQuery(dataset_id, config?.data_observability?.data_out_query_time_period); const [totalEventsResponse, totalFailedEventsResponse, totalApiCallsResponse, totalCallsAtDatasetLevelResponse] = await Promise.all([ axios.post(nativeQueryEndpoint, totalEventsPayload), From b68fb4433e3238ec106763a9599a4789d2dab60f Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 14 Jan 2025 16:29:51 +0530 Subject: [PATCH 286/311] feat: #OBS-I321: Removed comments --- api-service/src/controllers/TableMetrics/metricsHelper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/TableMetrics/metricsHelper.ts b/api-service/src/controllers/TableMetrics/metricsHelper.ts index 58e8d386..fd831c09 100644 --- a/api-service/src/controllers/TableMetrics/metricsHelper.ts +++ b/api-service/src/controllers/TableMetrics/metricsHelper.ts @@ -139,7 +139,7 @@ export const handleDataVolume = async (dataset_id: string, volume_by_days: numbe { type: "volume_percentage_by_hour", value: volumePercentageByHour }, { type: "volume_percentage_by_day", value: volumePercentageByDay }, { type: "volume_percentage_by_week", value: volumePercentageByWeek }, - { type: "growth_rate_percentage", value: volumePercentageByHour } // Assuming growth rate is same as volume percentage by hour + { type: "growth_rate_percentage", value: volumePercentageByHour } ] }; }; From c046cd8d6e086e0e896349c10f8c2c48dfabea00 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 14 Jan 2025 18:09:12 +0530 Subject: [PATCH 287/311] feat: #OBS-I321 : added volume by days default value --- api-service/src/controllers/TableMetrics/Metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/TableMetrics/Metrics.ts b/api-service/src/controllers/TableMetrics/Metrics.ts index 513414c0..236c1a5c 100644 --- a/api-service/src/controllers/TableMetrics/Metrics.ts +++ b/api-service/src/controllers/TableMetrics/Metrics.ts @@ -14,7 +14,7 @@ const tableMetrics = async (req: Request, res: Response) => { const msgid = _.get(req, "body.params.msgid"); const requestBody = req.body; - const { category, volume_by_days }: any = req.body.request; + const { category, volume_by_days = 10 }: any = req.body.request; const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; const startDate = '2000-01-01T00:00:00+05:30'; From b40bf3698b98c9854dc75ef8c3eae3825fc6b7b8 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 10:41:00 +0530 Subject: [PATCH 288/311] feat: #OBS-I321 : added default to importance score --- api-service/src/controllers/TableMetrics/metricsHelper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/TableMetrics/metricsHelper.ts b/api-service/src/controllers/TableMetrics/metricsHelper.ts index fd831c09..dd93ce78 100644 --- a/api-service/src/controllers/TableMetrics/metricsHelper.ts +++ b/api-service/src/controllers/TableMetrics/metricsHelper.ts @@ -83,7 +83,7 @@ export const handleDataObservability = async (dataset_id: string, intervals: str }, { type: "importance_score", - value: importanceScore + value: importanceScore || 0 } ] }; From 9dd8bb3ffa896c858b529af09f3da4cd580f78d7 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 11:38:35 +0530 Subject: [PATCH 289/311] feat: #OBS-I321 : Renamed file names --- .../DataMetrics/DataMetricsController.ts | 38 ++++++++ .../DataMetricsValidationSchema.json} | 0 .../DatasetMetrics.json} | 8 +- .../DatasetMetricsController.ts | 92 ++++++++++++++----- .../datasetMetricsHelper.ts} | 0 .../queries.ts | 0 .../src/controllers/TableMetrics/Metrics.ts | 72 --------------- api-service/src/routes/Router.ts | 8 +- 8 files changed, 115 insertions(+), 103 deletions(-) create mode 100644 api-service/src/controllers/DataMetrics/DataMetricsController.ts rename api-service/src/controllers/{DatasetMetrics/DatasetMetricsValidationSchema.json => DataMetrics/DataMetricsValidationSchema.json} (100%) rename api-service/src/controllers/{TableMetrics/TableMetrics.json => DatasetMetrics/DatasetMetrics.json} (88%) rename api-service/src/controllers/{TableMetrics/metricsHelper.ts => DatasetMetrics/datasetMetricsHelper.ts} (100%) rename api-service/src/controllers/{TableMetrics => DatasetMetrics}/queries.ts (100%) delete mode 100644 api-service/src/controllers/TableMetrics/Metrics.ts diff --git a/api-service/src/controllers/DataMetrics/DataMetricsController.ts b/api-service/src/controllers/DataMetrics/DataMetricsController.ts new file mode 100644 index 00000000..98314878 --- /dev/null +++ b/api-service/src/controllers/DataMetrics/DataMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DataMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") +} + +const dataMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, "data"); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default dataMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json b/api-service/src/controllers/DataMetrics/DataMetricsValidationSchema.json similarity index 100% rename from api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json rename to api-service/src/controllers/DataMetrics/DataMetricsValidationSchema.json diff --git a/api-service/src/controllers/TableMetrics/TableMetrics.json b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json similarity index 88% rename from api-service/src/controllers/TableMetrics/TableMetrics.json rename to api-service/src/controllers/DatasetMetrics/DatasetMetrics.json index 79923de8..c074544c 100644 --- a/api-service/src/controllers/TableMetrics/TableMetrics.json +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json @@ -4,7 +4,7 @@ "id": { "type": "string", "enum": [ - "api.table.metrics" + "api.dataset.metrics" ] }, "ver": { @@ -28,6 +28,9 @@ "request": { "type": "object", "properties": { + "dataset_id": { + "type": "string" + }, "category": { "type": "array", "items": { @@ -49,7 +52,8 @@ } }, "required": [ - "category" + "category", + "dataset_id" ] } }, diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 903d393c..e26c41c3 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -1,38 +1,80 @@ import { Request, Response } from "express"; -import _ from "lodash"; -import { executeNativeQuery } from "../../connections/druidConnection"; +import * as _ from "lodash" import { ResponseHandler } from "../../helpers/ResponseHandler"; -import vaidationSchema from "./DatasetMetricsValidationSchema.json" -import { schemaValidation } from "../../services/ValidationService"; +import dayjs from 'dayjs'; +import { handleConnectors, handleDataFreshness, handleDataLineage, handleDataObservability, handleDataQuality, handleDataVolume } from "./datasetMetricsHelper"; import logger from "../../logger"; -import { obsrvError } from "../../types/ObsrvError"; -import axios from "axios"; +import { schemaValidation } from "../../services/ValidationService"; +import validationSchema from "./DatasetMetrics.json"; import { config } from "../../configs/Config"; +import { datasetService } from "../../services/DatasetService"; -const getBaseUrl = (url: string) => { - if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") -} - +const apiId = "api.dataset.metrics"; const datasetMetrics = async (req: Request, res: Response) => { - const isValidSchema = schemaValidation(req.body, vaidationSchema); + const msgid = _.get(req, "body.params.msgid"); + const requestBody = req.body; + const dataset_id = _.get(req, "body.request.dataset_id"); + + const { category, volume_by_days = 10 }: any = req.body.request; + const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds + const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; + const startDate = '2000-01-01T00:00:00+05:30'; + const endDate = dayjs().add(1, 'day').format(dateFormat); + const intervals = `${startDate}/${endDate}`; + const isValidSchema = schemaValidation(requestBody, validationSchema); + const results = []; + if (!isValidSchema?.isValid) { - logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) - throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + logger.error({ apiId, datasetId: dataset_id, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); + } + + const dataset = await datasetService.getDataset(dataset_id, ["id"], true) + if (!dataset) { + logger.error({ apiId, message: `Dataset with id ${dataset_id} not found in live table`, code: "DATASET_NOT_FOUND" }) + return ResponseHandler.errorResponse({ message: `Dataset with id ${dataset_id} not found in live table`, code: "DATASET_NOT_FOUND", statusCode: 404, errCode: "NOT_FOUND" }, req, res); } - const { query } = req.body || {}; - const endpoint = query.url; - if (_.startsWith(endpoint, "/prom")) { - query.url = getBaseUrl(endpoint) - const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; - const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) - const data = _.get(apiResponse, "data"); - return res.json(data); + + try { + if (!category || category.includes("data_freshness")) { + const dataFreshnessResult = await handleDataFreshness(dataset_id, intervals, defaultThreshold); + results.push(dataFreshnessResult); + } + + if (!category || category.includes("data_observability")) { + const dataObservabilityResult = await handleDataObservability(dataset_id, intervals); + results.push(dataObservabilityResult); + } + + if (!category || category.includes("data_volume")) { + const dataVolumeResult = await handleDataVolume(dataset_id, volume_by_days, dateFormat); + results.push(dataVolumeResult); + } + + if (!category || category.includes("data_lineage")) { + const dataLineageResult = await handleDataLineage(dataset_id, intervals); + results.push(dataLineageResult); + } + + if (!category || category.includes("connectors")) { + const connectorsResult = await handleConnectors(dataset_id, intervals); + results.push(connectorsResult); + } + + if (!category || category.includes("data_quality")) { + const connectorsResult = await handleDataQuality(dataset_id, intervals); + results.push(connectorsResult); + } + + logger.info({ apiId, msgid, requestBody, datasetId: dataset_id, message: "Metrics fetched successfully" }) + return ResponseHandler.successResponse(req, res, { status: 200, data: results }); + } - else { - const query = _.get(req, ["body", "query", "body", "query"]); - const response = await executeNativeQuery(query); - ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + catch (error: any) { + logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS", error }); + return ResponseHandler.errorResponse({ message: "Error while fetching metrics", statusCode: 500, errCode: "FAILED", code: "FAILED_TO_FETCH_METRICS" }, req, res); } + } export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/TableMetrics/metricsHelper.ts b/api-service/src/controllers/DatasetMetrics/datasetMetricsHelper.ts similarity index 100% rename from api-service/src/controllers/TableMetrics/metricsHelper.ts rename to api-service/src/controllers/DatasetMetrics/datasetMetricsHelper.ts diff --git a/api-service/src/controllers/TableMetrics/queries.ts b/api-service/src/controllers/DatasetMetrics/queries.ts similarity index 100% rename from api-service/src/controllers/TableMetrics/queries.ts rename to api-service/src/controllers/DatasetMetrics/queries.ts diff --git a/api-service/src/controllers/TableMetrics/Metrics.ts b/api-service/src/controllers/TableMetrics/Metrics.ts deleted file mode 100644 index 236c1a5c..00000000 --- a/api-service/src/controllers/TableMetrics/Metrics.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Request, Response } from "express"; -import * as _ from "lodash" -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import dayjs from 'dayjs'; -import { handleConnectors, handleDataFreshness, handleDataLineage, handleDataObservability, handleDataQuality, handleDataVolume } from "./metricsHelper"; -import logger from "../../logger"; -import { schemaValidation } from "../../services/ValidationService"; -import validationSchema from "./TableMetrics.json"; -import { config } from "../../configs/Config"; - -const apiId = "api.table.metrics"; -const tableMetrics = async (req: Request, res: Response) => { - const { dataset_id } = req.params; - const msgid = _.get(req, "body.params.msgid"); - const requestBody = req.body; - - const { category, volume_by_days = 10 }: any = req.body.request; - const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds - const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; - const startDate = '2000-01-01T00:00:00+05:30'; - const endDate = dayjs().add(1, 'day').format(dateFormat); - const intervals = `${startDate}/${endDate}`; - const isValidSchema = schemaValidation(requestBody, validationSchema); - const results = []; - if (!isValidSchema?.isValid) { - logger.error({ apiId, datasetId:dataset_id, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); - } - - try { - if (!category || category.includes("data_freshness")) { - const dataFreshnessResult = await handleDataFreshness(dataset_id, intervals, defaultThreshold); - results.push(dataFreshnessResult); - } - - if (!category || category.includes("data_observability")) { - const dataObservabilityResult = await handleDataObservability(dataset_id, intervals); - results.push(dataObservabilityResult); - } - - if (!category || category.includes("data_volume")) { - const dataVolumeResult = await handleDataVolume(dataset_id, volume_by_days, dateFormat); - results.push(dataVolumeResult); - } - - if (!category || category.includes("data_lineage")) { - const dataLineageResult = await handleDataLineage(dataset_id, intervals); - results.push(dataLineageResult); - } - - if (!category || category.includes("connectors")) { - const connectorsResult = await handleConnectors(dataset_id, intervals); - results.push(connectorsResult); - } - - if (!category || category.includes("data_quality")) { - const connectorsResult = await handleDataQuality(dataset_id, intervals); - results.push(connectorsResult); - } - - logger.info({ apiId, msgid, requestBody, datasetId: dataset_id, message: "Metrics fetched successfully" }) - return ResponseHandler.successResponse(req, res, { status: 200, data: results }); - - } - catch (error: any) { - logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS", error }); - return ResponseHandler.errorResponse({ message: "Error while fetching metrics", statusCode: 500, errCode: "FAILED", code: "FAILED_TO_FETCH_METRICS" }, req, res); - } - -} - -export default tableMetrics; \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index fb94237e..3cb2fcf6 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,10 +29,10 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; -import tableMetrics from "../controllers/TableMetrics/Metrics"; +import dataMetrics from "../controllers/DataMetrics/DataMetricsController"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; export const router = express.Router(); @@ -63,5 +63,5 @@ router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), o router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); -router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) -router.post("/table/metrics/:dataset_id",setDataToRequestObject("api.table.metrics"), tableMetrics); +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), dataMetrics) +router.post("/dataset/metrics", setDataToRequestObject("api.dataset.metrics"), checkRBAC.handler(), datasetMetrics); From 439198563908b0e609c13dffe9d3f83b8dcd0b6f Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 11:43:33 +0530 Subject: [PATCH 290/311] feat: #OBS-I321 : Added dataset metrics as general access --- api-service/src/middlewares/userPermissions.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index 400e218f..1b4e6b68 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -12,7 +12,8 @@ "api.alert.silence.list", "api.alert.silence.get", "api.alert.notification.list", - "api.alert.notification.get" + "api.alert.notification.get", + "api.dataset.metrics" ], "restricted_dataset_api": [ "api.datasets.reset", From 96ec165bec73ec8127bf9bd6b800fdf6a6bebc62 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 11:58:47 +0530 Subject: [PATCH 291/311] feat: #OBS-I321 : added defaults --- api-service/src/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 26a64a58..7cd2c5f1 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -121,7 +121,7 @@ export const config = { }, "storage_types": process.env.storage_types || '{"lake_house":true,"realtime_store":true}', "data_observability": { - "default_freshness_threshold": process.env.default_freshness_threshold || 5, // in minutes + "default_freshness_threshold": process.env.default_freshness_threshold ? parseInt(process.env.default_freshness_threshold) : 5, // in minutes "data_out_query_time_period": process.env.data_out_query_time_period || "1d", } } From c23147be61a6ef6cb71a47d0f52dacfec5369554 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:08:43 +0530 Subject: [PATCH 292/311] feat: #OBS-I321 : changed function parameter name --- .../src/controllers/DatasetMetrics/queries.ts | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/queries.ts b/api-service/src/controllers/DatasetMetrics/queries.ts index 41b72cdc..04701907 100644 --- a/api-service/src/controllers/DatasetMetrics/queries.ts +++ b/api-service/src/controllers/DatasetMetrics/queries.ts @@ -1,6 +1,6 @@ import dayjs from "dayjs"; -export const processingTimeQuery = (intervals: string, dataset: string) => ({ +export const processingTimeQuery = (intervals: string, dataset_id: string) => ({ query: { queryType: "groupBy", dataSource: "system-events", @@ -13,7 +13,7 @@ export const processingTimeQuery = (intervals: string, dataset: string) => ({ type: "and", fields: [ { type: "selector", dimension: "ctx_module", value: "processing" }, - { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, { type: "selector", dimension: "error_code", value: null } ] @@ -32,7 +32,7 @@ export const processingTimeQuery = (intervals: string, dataset: string) => ({ } }); -export const totalEventsQuery = (intervals: string, dataset: string) => ({ +export const totalEventsQuery = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: { type: "table", @@ -46,7 +46,7 @@ export const totalEventsQuery = (intervals: string, dataset: string) => ({ type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id }, granularity: { type: "all" @@ -60,7 +60,7 @@ export const totalEventsQuery = (intervals: string, dataset: string) => ({ ] }); -export const totalFailedEventsQuery = (intervals: string, dataset: string) => ({ +export const totalFailedEventsQuery = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: { type: "table", @@ -74,7 +74,7 @@ export const totalFailedEventsQuery = (intervals: string, dataset: string) => ({ type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id }, granularity: { type: "all" @@ -109,7 +109,7 @@ export const totalFailedEventsQuery = (intervals: string, dataset: string) => ({ ] }); -export const generateTimeseriesQuery = (intervals: string, dataset: string) => ({ +export const generateTimeseriesQuery = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: "system-events", intervals: intervals, @@ -121,7 +121,7 @@ export const generateTimeseriesQuery = (intervals: string, dataset: string) => ( type: "and", fields: [ { type: "selector", dimension: "ctx_module", value: "processing" }, - { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, { type: "selector", dimension: "error_code", value: null } ] @@ -131,7 +131,7 @@ export const generateTimeseriesQuery = (intervals: string, dataset: string) => ( ] }); -export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset: string) => ({ +export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: "system-events", intervals: intervals, @@ -143,7 +143,7 @@ export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset: type: "and", fields: [ { type: "selector", dimension: "ctx_module", value: "processing" }, - { type: "selector", dimension: "ctx_dataset", value: dataset }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, { type: "selector", dimension: "error_code", value: null } ] @@ -153,7 +153,7 @@ export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset: ] }); -export const dataLineageSuccessQuery = (intervals: string, dataset: string, column: string, value: string) => ({ +export const dataLineageSuccessQuery = (intervals: string, dataset_id: string, column: string, value: string) => ({ queryType: "timeseries", dataSource: { type: "table", @@ -176,7 +176,7 @@ export const dataLineageSuccessQuery = (intervals: string, dataset: string, colu type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id } ] }, @@ -192,7 +192,7 @@ export const dataLineageSuccessQuery = (intervals: string, dataset: string, colu ] }); -export const generateTransformationFailedQuery = (intervals: string, dataset: string) => ({ +export const generateTransformationFailedQuery = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: { type: "table", @@ -206,7 +206,7 @@ export const generateTransformationFailedQuery = (intervals: string, dataset: st type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id }, granularity: { type: "all" @@ -241,7 +241,7 @@ export const generateTransformationFailedQuery = (intervals: string, dataset: st ] }); -export const generateDedupFailedQuery = (intervals: string, dataset: string) => ({ +export const generateDedupFailedQuery = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: { type: "table", @@ -255,7 +255,7 @@ export const generateDedupFailedQuery = (intervals: string, dataset: string) => type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id }, granularity: { type: "all" @@ -290,7 +290,7 @@ export const generateDedupFailedQuery = (intervals: string, dataset: string) => ] }); -export const generateDenormFailedQuery = (intervals: string, dataset: string) => ({ +export const generateDenormFailedQuery = (intervals: string, dataset_id: string) => ({ queryType: "timeseries", dataSource: { type: "table", @@ -304,7 +304,7 @@ export const generateDenormFailedQuery = (intervals: string, dataset: string) => type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id }, granularity: { type: "all" @@ -339,7 +339,7 @@ export const generateDenormFailedQuery = (intervals: string, dataset: string) => ] }); -export const generateConnectorQuery = (intervals: string, dataset: string) => ({ +export const generateConnectorQuery = (intervals: string, dataset_id: string) => ({ queryType: "topN", dataSource: { type: "table", @@ -366,7 +366,7 @@ export const generateConnectorQuery = (intervals: string, dataset: string) => ({ type: "equals", column: "ctx_dataset", matchValueType: "STRING", - matchValue: dataset + matchValue: dataset_id }, granularity: { type: "all" From cb0c60a8b763b3981b15cf5417b89d72750ed5d0 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 15 Jan 2025 12:40:18 +0530 Subject: [PATCH 293/311] feat #OBS-I501 : data analyse PII API added --- api-service/src/configs/Config.ts | 2 +- .../connections/commandServiceConnection.ts | 18 +++++++++---- .../ConnectorRegisterController.ts | 3 ++- .../DataAnalyzePIIController.ts | 26 +++++++++++++++++++ .../controllers/DatasetReset/DatasetReset.ts | 3 ++- .../DatasetStatusTransition.ts | 17 ++++++------ .../src/middlewares/userPermissions.json | 12 ++++++--- api-service/src/routes/Router.ts | 2 ++ api-service/src/services/DatasetService.ts | 4 +-- 9 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index a2aa79a1..18912018 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -98,7 +98,7 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), - "path": process.env.command_service_path || "/system/v1/dataset/command" + "paths": JSON.parse(process.env.command_service_paths || '{"dataset":"/system/v1/dataset/command","connector":"/connector/v1/register","analyzePII":"/system/data/v1/analyze/pii"}') }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index fc77fc77..c6f1b411 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -5,11 +5,14 @@ import { v4 } from "uuid"; const commandHost = _.get(config, ["command_service_config", "host"]) const commandPort = _.get(config, ["command_service_config", "port"]) -const commandPath = _.get(config, ["command_service_config", "path"]) +const commandPaths = _.get(config, ["command_service_config", "paths"]) +const datasetPath = _.get(commandPaths, ["dataset"]) +const connectorRegisterPath = _.get(commandPaths, ["connector"]) +const analyzePIIPath = _.get(commandPaths, ["analyzePII"]) export const commandHttpService = axios.create({ baseURL: `${commandHost}:${commandPort}`, headers: { "Content-Type": "application/json" } }); -export const executeCommand = async (id: string, command: string) => { +export const executeCommand = async (id: string, command: string, userToken: string) => { const payload = { "id": v4(), "data": { @@ -17,9 +20,14 @@ export const executeCommand = async (id: string, command: string) => { "command": command } } - return commandHttpService.post(commandPath, payload) + return commandHttpService.post(datasetPath, payload, { headers: { Authorization: userToken }}) } -export const registerConnector = async (requestBody: any) => { - return commandHttpService.post("/connector/v1/register", requestBody) +export const registerConnector = async (requestBody: any, userToken: string) => { + return commandHttpService.post(connectorRegisterPath, requestBody, { headers: { Authorization: userToken }}) +} + +export const detectPII = async (requestBody: any, userToken: string) => { + console.log(`analyzePIIPath : ${analyzePIIPath}`) + return commandHttpService.post(analyzePIIPath, requestBody, { headers: { Authorization: userToken }}) } \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts index 4844ac7e..4ab7c05f 100644 --- a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -22,7 +22,8 @@ const connectorRegisterController = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await registerConnector(payload); + const userToken = req.get('authorization') as string; + const registryResponse = await registerConnector(payload, userToken); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { diff --git a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts new file mode 100644 index 00000000..50251e91 --- /dev/null +++ b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts @@ -0,0 +1,26 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import * as _ from "lodash"; +import logger from "../../logger"; +import { detectPII } from "../../connections/commandServiceConnection"; + +const code = "FAILED_TO_DETECT_PII"; +export const dataAnalyzePII = async (req: Request, res: Response) => { + const apiId = _.get(req, 'id') + try { + const userToken = req.get('authorization') as string; + const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'data']), userToken); + logger.info({apiId , message: `Detected PII successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: piiSuggestionsResponse?.data}) + } catch (error: any) { + const errMessage = _.get(error, "response.data.detail") + logger.error(error, apiId, code); + let errorMessage = error; + const statusCode = _.get(error, "status") + if (!statusCode || statusCode == 500) { + errorMessage = { code, message: errMessage || "Failed to detect pii" } + } + ResponseHandler.errorResponse(errorMessage, req, res); + } +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index 68b272f1..3f1d5922 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -29,12 +29,13 @@ const datasetReset = async (req: Request, res: Response) => { const category = _.get(req, ["body", "request", "category"]); const datasetId = _.get(req, ["params"," datasetId"]); + const userToken = req.get('authorization') as string; await validateRequest(req); if (category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() if (pipeLineStatus == HealthStatus.UnHealthy) { - await restartPipeline({ "dataset": { "dataset_id": datasetId } }) + await restartPipeline({ "dataset": { "dataset_id": datasetId } }, userToken) } } else if (category == "query") { const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index ff1502d7..e09fa5b8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -50,6 +50,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); + const userToken = req.get('authorization') as string; validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) @@ -64,10 +65,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userID); + await publishDataset(dataset, userID, userToken); break; case "Retire": - await retireDataset(dataset, userID); + await retireDataset(dataset, userID, userToken); break; default: throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) @@ -134,7 +135,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userID: any) => { +const publishDataset = async (dataset: Record, userID: any, userToken: string) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record validateStorageSupport(draftDataset); @@ -142,7 +143,7 @@ const publishDataset = async (dataset: Record, userID: any) => { _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) - await datasetService.publishDataset(draftDataset) + await datasetService.publishDataset(draftDataset, userToken); } const validateAndUpdateDenormConfig = async (draftDataset: Record) => { @@ -227,11 +228,11 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record, updated_by: any) => { +const retireDataset = async (dataset: Record, updated_by: any, userToken: string) => { await canRetireIfMasterDataset(dataset); await datasetService.retireDataset(dataset, updated_by); - await restartPipeline(dataset); + await restartPipeline(dataset, userToken); } @@ -254,8 +255,8 @@ const canRetireIfMasterDataset = async (dataset: Record) => { } } -export const restartPipeline = async (dataset: Record) => { - return executeCommand(dataset.id, "RESTART_PIPELINE") +export const restartPipeline = async (dataset: Record, userToken: string) => { + return executeCommand(dataset.id, "RESTART_PIPELINE", userToken) } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index eda444c4..c2c1f54c 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -50,7 +50,8 @@ "api.datasets.dataschema" ], "data": [ - "api.data.in" + "api.data.in", + "api.data.analyze.pii" ], "queryTemplate": [ "api.query.template.create", @@ -92,7 +93,8 @@ "schema", "file", "connector", - "sqlQuery" + "sqlQuery", + "data" ], "dataset_manager": [ "general_access", @@ -104,7 +106,8 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "data" ], "admin": [ "general_access", @@ -120,7 +123,8 @@ "alert", "metric", "silence", - "notificationChannel" + "notificationChannel", + "data" ], "operations_admin": [ "alert", diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1e39d942..a3c13e1e 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -32,6 +32,7 @@ import telemetryActions from "../telemetry/telemetryActions"; import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; +import { dataAnalyzePII } from "../controllers/DataAnalyzePII/DataAnalyzePIIController"; export const router = express.Router(); @@ -60,6 +61,7 @@ router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), o router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); +router.post("/data/analyze/pii", setDataToRequestObject("api.data.analyze.pii"), onRequest({ entity: Entity.Management }),checkRBAC.handler(), dataAnalyzePII); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index a5821892..b953b47f 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -303,7 +303,7 @@ class DatasetService { } } - publishDataset = async (draftDataset: Record) => { + publishDataset = async (draftDataset: Record, userToken: string) => { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() @@ -326,7 +326,7 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET", userToken); } From cc5fc4ae80c1003c8a8772d2e26fe03dfffc373e Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 15 Jan 2025 12:41:57 +0530 Subject: [PATCH 294/311] Revert "feat #OBS-I501 : data analyse PII API added" This reverts commit cb0c60a8b763b3981b15cf5417b89d72750ed5d0. --- api-service/src/configs/Config.ts | 2 +- .../connections/commandServiceConnection.ts | 18 ++++--------- .../ConnectorRegisterController.ts | 3 +-- .../DataAnalyzePIIController.ts | 26 ------------------- .../controllers/DatasetReset/DatasetReset.ts | 3 +-- .../DatasetStatusTransition.ts | 17 ++++++------ .../src/middlewares/userPermissions.json | 12 +++------ api-service/src/routes/Router.ts | 2 -- api-service/src/services/DatasetService.ts | 4 +-- 9 files changed, 22 insertions(+), 65 deletions(-) delete mode 100644 api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 18912018..a2aa79a1 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -98,7 +98,7 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), - "paths": JSON.parse(process.env.command_service_paths || '{"dataset":"/system/v1/dataset/command","connector":"/connector/v1/register","analyzePII":"/system/data/v1/analyze/pii"}') + "path": process.env.command_service_path || "/system/v1/dataset/command" }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index c6f1b411..fc77fc77 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -5,14 +5,11 @@ import { v4 } from "uuid"; const commandHost = _.get(config, ["command_service_config", "host"]) const commandPort = _.get(config, ["command_service_config", "port"]) -const commandPaths = _.get(config, ["command_service_config", "paths"]) -const datasetPath = _.get(commandPaths, ["dataset"]) -const connectorRegisterPath = _.get(commandPaths, ["connector"]) -const analyzePIIPath = _.get(commandPaths, ["analyzePII"]) +const commandPath = _.get(config, ["command_service_config", "path"]) export const commandHttpService = axios.create({ baseURL: `${commandHost}:${commandPort}`, headers: { "Content-Type": "application/json" } }); -export const executeCommand = async (id: string, command: string, userToken: string) => { +export const executeCommand = async (id: string, command: string) => { const payload = { "id": v4(), "data": { @@ -20,14 +17,9 @@ export const executeCommand = async (id: string, command: string, userToken: str "command": command } } - return commandHttpService.post(datasetPath, payload, { headers: { Authorization: userToken }}) + return commandHttpService.post(commandPath, payload) } -export const registerConnector = async (requestBody: any, userToken: string) => { - return commandHttpService.post(connectorRegisterPath, requestBody, { headers: { Authorization: userToken }}) -} - -export const detectPII = async (requestBody: any, userToken: string) => { - console.log(`analyzePIIPath : ${analyzePIIPath}`) - return commandHttpService.post(analyzePIIPath, requestBody, { headers: { Authorization: userToken }}) +export const registerConnector = async (requestBody: any) => { + return commandHttpService.post("/connector/v1/register", requestBody) } \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts index 4ab7c05f..4844ac7e 100644 --- a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -22,8 +22,7 @@ const connectorRegisterController = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const userToken = req.get('authorization') as string; - const registryResponse = await registerConnector(payload, userToken); + const registryResponse = await registerConnector(payload); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { diff --git a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts deleted file mode 100644 index 50251e91..00000000 --- a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Request, Response } from "express"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import httpStatus from "http-status"; -import * as _ from "lodash"; -import logger from "../../logger"; -import { detectPII } from "../../connections/commandServiceConnection"; - -const code = "FAILED_TO_DETECT_PII"; -export const dataAnalyzePII = async (req: Request, res: Response) => { - const apiId = _.get(req, 'id') - try { - const userToken = req.get('authorization') as string; - const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'data']), userToken); - logger.info({apiId , message: `Detected PII successfully` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: piiSuggestionsResponse?.data}) - } catch (error: any) { - const errMessage = _.get(error, "response.data.detail") - logger.error(error, apiId, code); - let errorMessage = error; - const statusCode = _.get(error, "status") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: errMessage || "Failed to detect pii" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index 3f1d5922..68b272f1 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -29,13 +29,12 @@ const datasetReset = async (req: Request, res: Response) => { const category = _.get(req, ["body", "request", "category"]); const datasetId = _.get(req, ["params"," datasetId"]); - const userToken = req.get('authorization') as string; await validateRequest(req); if (category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() if (pipeLineStatus == HealthStatus.UnHealthy) { - await restartPipeline({ "dataset": { "dataset_id": datasetId } }, userToken) + await restartPipeline({ "dataset": { "dataset_id": datasetId } }) } } else if (category == "query") { const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index e09fa5b8..ff1502d7 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -50,7 +50,6 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); - const userToken = req.get('authorization') as string; validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) @@ -65,10 +64,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userID, userToken); + await publishDataset(dataset, userID); break; case "Retire": - await retireDataset(dataset, userID, userToken); + await retireDataset(dataset, userID); break; default: throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) @@ -135,7 +134,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userID: any, userToken: string) => { +const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record validateStorageSupport(draftDataset); @@ -143,7 +142,7 @@ const publishDataset = async (dataset: Record, userID: any, userTok _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) - await datasetService.publishDataset(draftDataset, userToken); + await datasetService.publishDataset(draftDataset) } const validateAndUpdateDenormConfig = async (draftDataset: Record) => { @@ -228,11 +227,11 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record, updated_by: any, userToken: string) => { +const retireDataset = async (dataset: Record, updated_by: any) => { await canRetireIfMasterDataset(dataset); await datasetService.retireDataset(dataset, updated_by); - await restartPipeline(dataset, userToken); + await restartPipeline(dataset); } @@ -255,8 +254,8 @@ const canRetireIfMasterDataset = async (dataset: Record) => { } } -export const restartPipeline = async (dataset: Record, userToken: string) => { - return executeCommand(dataset.id, "RESTART_PIPELINE", userToken) +export const restartPipeline = async (dataset: Record) => { + return executeCommand(dataset.id, "RESTART_PIPELINE") } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index c2c1f54c..eda444c4 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -50,8 +50,7 @@ "api.datasets.dataschema" ], "data": [ - "api.data.in", - "api.data.analyze.pii" + "api.data.in" ], "queryTemplate": [ "api.query.template.create", @@ -93,8 +92,7 @@ "schema", "file", "connector", - "sqlQuery", - "data" + "sqlQuery" ], "dataset_manager": [ "general_access", @@ -106,8 +104,7 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api", - "data" + "restricted_dataset_api" ], "admin": [ "general_access", @@ -123,8 +120,7 @@ "alert", "metric", "silence", - "notificationChannel", - "data" + "notificationChannel" ], "operations_admin": [ "alert", diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index a3c13e1e..1e39d942 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -32,7 +32,6 @@ import telemetryActions from "../telemetry/telemetryActions"; import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; -import { dataAnalyzePII } from "../controllers/DataAnalyzePII/DataAnalyzePIIController"; export const router = express.Router(); @@ -61,7 +60,6 @@ router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), o router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); -router.post("/data/analyze/pii", setDataToRequestObject("api.data.analyze.pii"), onRequest({ entity: Entity.Management }),checkRBAC.handler(), dataAnalyzePII); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index b953b47f..a5821892 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -303,7 +303,7 @@ class DatasetService { } } - publishDataset = async (draftDataset: Record, userToken: string) => { + publishDataset = async (draftDataset: Record) => { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() @@ -326,7 +326,7 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET", userToken); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); } From ed07dec147d684f282a7368289d5b985569910ad Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:48:36 +0530 Subject: [PATCH 295/311] feat: #OBS-I321 : changed datasetId to dataset_id in params --- .../controllers/DataExhaust/DataExhaustController.ts | 10 +++++----- .../DataIngestion/DataIngestionController.ts | 2 +- .../src/controllers/DataOut/DataOutController.ts | 2 +- .../src/controllers/DatasetReset/DatasetReset.ts | 4 ++-- api-service/src/routes/Router.ts | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api-service/src/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/controllers/DataExhaust/DataExhaustController.ts index ceeb3358..858cf233 100644 --- a/api-service/src/controllers/DataExhaust/DataExhaustController.ts +++ b/api-service/src/controllers/DataExhaust/DataExhaustController.ts @@ -11,7 +11,7 @@ import { cloudProvider } from "../../services/CloudServices"; export const dataExhaust = async (req: Request, res: Response) => { const { params } = req; - const { datasetId } = params; + const { dataset_id } = params; const { type }: any = req.query; const momentFormat = "YYYY-MM-DD"; @@ -28,12 +28,12 @@ export const dataExhaust = async (req: Request, res: Response) => { return resData || {}; } - if (type && config.cloud_config.exclude_exhaust_types.includes(datasetId)) { + if (type && config.cloud_config.exclude_exhaust_types.includes(dataset_id)) { return ResponseHandler.errorResponse({ statusCode: 404, message: "Record not found", errCode: httpStatus["404_NAME"] }, req, res) } - const datasetRecord = await verifyDatasetExists(datasetId); + const datasetRecord = await verifyDatasetExists(dataset_id); if (datasetRecord === null) { - logger.error(`Dataset with ${datasetId} not found in live table`) + logger.error(`Dataset with ${dataset_id} not found in live table`) return ResponseHandler.errorResponse({ statusCode: 404, message: "Record not found", errCode: httpStatus["404_NAME"] }, req, res) } const dateRange = getDateRange(req); @@ -47,7 +47,7 @@ export const dataExhaust = async (req: Request, res: Response) => { return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid date range! make sure your range cannot be more than ${config.cloud_config.maxQueryDateRange} days`, errCode: "BAD_REQUEST" }, req, res) } - const resData: any = await getFromStorage(type, dateRange, datasetId); + const resData: any = await getFromStorage(type, dateRange, dataset_id); if (_.isEmpty(resData.files)) { logger.error("Date range provided does not have any backup files") return ResponseHandler.errorResponse({ statusCode: 404, message: "Date range provided does not have any backup files", errCode: "NOT_FOUND" }, req, res); diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 5044bb43..39090dda 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -27,7 +27,7 @@ const apiId = "api.data.in"; const dataIn = async (req: Request, res: Response) => { const requestBody = req.body; - const datasetId = req.params.datasetId.trim(); + const datasetId = req.params.dataset_id.trim(); const isValidSchema = schemaValidation(requestBody, validationSchema) if (!isValidSchema?.isValid) { diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts index dd6f93b6..fa5eee51 100644 --- a/api-service/src/controllers/DataOut/DataOutController.ts +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -9,7 +9,7 @@ import { executeNativeQuery, executeSqlQuery } from "../../connections/druidConn export const apiId = "api.data.out"; const dataOut = async (req: Request, res: Response) => { - const datasetId = req.params?.datasetId; + const datasetId = req.params?.dataset_id; const requestBody = req.body; const msgid = _.get(req, "body.params.msgid"); const isValidSchema = schemaValidation(requestBody, validationSchema); diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index 68b272f1..cc5aef3c 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -18,7 +18,7 @@ const validateRequest = async (req: Request) => { if (!isRequestValid.isValid) { throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) } - const datasetId = _.get(req, ["params", "datasetId"]) + const datasetId = _.get(req, ["params", "dataset_id"]) const isDataSetExists = await datasetService.checkDatasetExists(datasetId); if (!isDataSetExists) { throw obsrvError(datasetId, "DATASET_NOT_FOUND", `Dataset not exists with id:${datasetId}`, httpStatus[httpStatus.NOT_FOUND], 404) @@ -28,7 +28,7 @@ const validateRequest = async (req: Request) => { const datasetReset = async (req: Request, res: Response) => { const category = _.get(req, ["body", "request", "category"]); - const datasetId = _.get(req, ["params"," datasetId"]); + const datasetId = _.get(req, ["params"," dataset_id"]); await validateRequest(req); if (category == "processing") { diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 3cb2fcf6..60930f05 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -36,13 +36,13 @@ import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsControll export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/data/in/:dataset_id", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:dataset_id", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.get("/data/exhaust/:dataset_id", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); @@ -53,7 +53,7 @@ router.post("/template/query/:templateId", setDataToRequestObject("api.query.tem router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); +router.post("/datasets/reset/:dataset_id", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); @@ -64,4 +64,4 @@ router.post("/connector/register", setDataToRequestObject("api.connector.registe //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), dataMetrics) -router.post("/dataset/metrics", setDataToRequestObject("api.dataset.metrics"), checkRBAC.handler(), datasetMetrics); +router.post("/dataset/metrics", setDataToRequestObject("api.dataset.metrics"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetMetrics); From e95ca5f70103a1ff9d61eb89596cf835d1ce66a5 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:49:08 +0530 Subject: [PATCH 296/311] feat: #OBS-I321 : Reverted to dataset_id from datasetID --- api-service/src/metrics/prometheus/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index febb5d59..05051461 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -58,7 +58,7 @@ const getMetricLabels = (req: any, res: Response) => { const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, ["body", "request", "datasetId"]) || _.get(req, ["params", "datasetId"]) || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null const duration = getDuration(startTime); const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } From cf0370f8604ed27c15eff954be725b16a9d9da82 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:49:35 +0530 Subject: [PATCH 297/311] feat: #OBS-I321 : Added dataset metric service file --- .../DatasetMetricsController.ts | 21 ++++++++++--------- .../DatasetMetricsService.ts} | 16 +++++++------- 2 files changed, 19 insertions(+), 18 deletions(-) rename api-service/src/{controllers/DatasetMetrics/datasetMetricsHelper.ts => services/DatasetMetricsService.ts} (95%) diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index e26c41c3..1af73eea 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -2,28 +2,29 @@ import { Request, Response } from "express"; import * as _ from "lodash" import { ResponseHandler } from "../../helpers/ResponseHandler"; import dayjs from 'dayjs'; -import { handleConnectors, handleDataFreshness, handleDataLineage, handleDataObservability, handleDataQuality, handleDataVolume } from "./datasetMetricsHelper"; import logger from "../../logger"; import { schemaValidation } from "../../services/ValidationService"; import validationSchema from "./DatasetMetrics.json"; import { config } from "../../configs/Config"; import { datasetService } from "../../services/DatasetService"; +import { getConnectors, getDataFreshness, getDataLineage, getDataObservability, getDataQuality, getDataVolume } from "../../services/DatasetMetricsService"; const apiId = "api.dataset.metrics"; const datasetMetrics = async (req: Request, res: Response) => { const msgid = _.get(req, "body.params.msgid"); const requestBody = req.body; const dataset_id = _.get(req, "body.request.dataset_id"); + const startDateValue = _.get(req, "body.request.start_date") || config?.data_observability?.default_query_time_period; const { category, volume_by_days = 10 }: any = req.body.request; const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; - const startDate = '2000-01-01T00:00:00+05:30'; - const endDate = dayjs().add(1, 'day').format(dateFormat); + const endDate = dayjs().format(dateFormat); + const startDate = dayjs(endDate).subtract(startDateValue, 'day').format(dateFormat); const intervals = `${startDate}/${endDate}`; const isValidSchema = schemaValidation(requestBody, validationSchema); const results = []; - + if (!isValidSchema?.isValid) { logger.error({ apiId, datasetId: dataset_id, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); @@ -37,32 +38,32 @@ const datasetMetrics = async (req: Request, res: Response) => { try { if (!category || category.includes("data_freshness")) { - const dataFreshnessResult = await handleDataFreshness(dataset_id, intervals, defaultThreshold); + const dataFreshnessResult = await getDataFreshness(dataset_id, intervals, defaultThreshold); results.push(dataFreshnessResult); } if (!category || category.includes("data_observability")) { - const dataObservabilityResult = await handleDataObservability(dataset_id, intervals); + const dataObservabilityResult = await getDataObservability(dataset_id, intervals); results.push(dataObservabilityResult); } if (!category || category.includes("data_volume")) { - const dataVolumeResult = await handleDataVolume(dataset_id, volume_by_days, dateFormat); + const dataVolumeResult = await getDataVolume(dataset_id, volume_by_days, dateFormat); results.push(dataVolumeResult); } if (!category || category.includes("data_lineage")) { - const dataLineageResult = await handleDataLineage(dataset_id, intervals); + const dataLineageResult = await getDataLineage(dataset_id, intervals); results.push(dataLineageResult); } if (!category || category.includes("connectors")) { - const connectorsResult = await handleConnectors(dataset_id, intervals); + const connectorsResult = await getConnectors(dataset_id, intervals); results.push(connectorsResult); } if (!category || category.includes("data_quality")) { - const connectorsResult = await handleDataQuality(dataset_id, intervals); + const connectorsResult = await getDataQuality(dataset_id, intervals); results.push(connectorsResult); } diff --git a/api-service/src/controllers/DatasetMetrics/datasetMetricsHelper.ts b/api-service/src/services/DatasetMetricsService.ts similarity index 95% rename from api-service/src/controllers/DatasetMetrics/datasetMetricsHelper.ts rename to api-service/src/services/DatasetMetricsService.ts index dd93ce78..2c879d64 100644 --- a/api-service/src/controllers/DatasetMetrics/datasetMetricsHelper.ts +++ b/api-service/src/services/DatasetMetricsService.ts @@ -1,14 +1,14 @@ import axios from "axios"; import dayjs from "dayjs"; import _ from "lodash"; -import { config } from "../../configs/Config"; -import { dataLineageSuccessQuery, generateConnectorQuery, generateDatasetQueryCallsQuery, generateDedupFailedQuery, generateDenormFailedQuery, generateTimeseriesQuery, generateTimeseriesQueryEventsPerHour, generateTotalQueryCallsQuery, generateTransformationFailedQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "./queries"; +import { config } from "../configs/Config"; +import { dataLineageSuccessQuery, generateConnectorQuery, generateDatasetQueryCallsQuery, generateDedupFailedQuery, generateDenormFailedQuery, generateTimeseriesQuery, generateTimeseriesQueryEventsPerHour, generateTotalQueryCallsQuery, generateTransformationFailedQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "../controllers/DatasetMetrics/queries"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; const prometheusEndpoint = `${config.query_api.prometheus.url}/api/v1/query_range`; -export const handleDataFreshness = async (dataset_id: string, intervals: string, defaultThreshold: number) => { +export const getDataFreshness = async (dataset_id: string, intervals: string, defaultThreshold: number) => { const queryPayload = processingTimeQuery(intervals, dataset_id); const druidResponse = await axios.post(nativeQueryEndpoint, queryPayload?.query); const avgProcessingTime = _.get(druidResponse, "data[0].average_processing_time", 0); @@ -34,7 +34,7 @@ export const handleDataFreshness = async (dataset_id: string, intervals: string, }; }; -export const handleDataObservability = async (dataset_id: string, intervals: string) => { +export const getDataObservability = async (dataset_id: string, intervals: string) => { const totalEventsPayload = totalEventsQuery(intervals, dataset_id); const totalFailedEventsPayload = totalFailedEventsQuery(intervals, dataset_id); const totalQueryCalls = generateTotalQueryCallsQuery(config?.data_observability?.data_out_query_time_period); @@ -89,7 +89,7 @@ export const handleDataObservability = async (dataset_id: string, intervals: str }; }; -export const handleDataVolume = async (dataset_id: string, volume_by_days: number, dateFormat: string) => { +export const getDataVolume = async (dataset_id: string, volume_by_days: number, dateFormat: string) => { const currentHourIntervals = dayjs().subtract(1, "hour").startOf("hour").toISOString() + "/" + dayjs().startOf("hour").toISOString(); const currentDayIntervals = dayjs().subtract(1, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); const currentWeekIntervals = dayjs().subtract(1, 'week').startOf('week').format(dateFormat) + '/' + dayjs().endOf('week').format(dateFormat); @@ -144,7 +144,7 @@ export const handleDataVolume = async (dataset_id: string, volume_by_days: numbe }; }; -export const handleDataLineage = async (dataset_id: string, intervals: string) => { +export const getDataLineage = async (dataset_id: string, intervals: string) => { const transformationSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "transformer_status", "success"); const dedupSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "dedup_status", "success"); const denormSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "denorm_status", "success"); @@ -196,7 +196,7 @@ export const handleDataLineage = async (dataset_id: string, intervals: string) = }; -export const handleConnectors = async (dataset_id: string, intervals: string) => { +export const getConnectors = async (dataset_id: string, intervals: string) => { const connectorQueryPayload = generateConnectorQuery(intervals, dataset_id); const connectorResponse = await axios.post(nativeQueryEndpoint, connectorQueryPayload); const connectorsData = _.get(connectorResponse, "data[0].result", []); @@ -215,7 +215,7 @@ export const handleConnectors = async (dataset_id: string, intervals: string) => }; }; -export const handleDataQuality = async (dataset_id: string, intervals: string) => { +export const getDataQuality = async (dataset_id: string, intervals: string) => { const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); const [totalValidationResponse, totalValidationFailedResponse, From 89bfd3b869881ec11de72bfc8182ae634c0c2ec5 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:50:16 +0530 Subject: [PATCH 298/311] feat: #OBS-I321 : Added default query time period --- api-service/src/configs/Config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 7cd2c5f1..702cc13f 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -123,5 +123,6 @@ export const config = { "data_observability": { "default_freshness_threshold": process.env.default_freshness_threshold ? parseInt(process.env.default_freshness_threshold) : 5, // in minutes "data_out_query_time_period": process.env.data_out_query_time_period || "1d", + "default_query_time_period": process.env.default_query_time_period ? parseInt(process.env.default_query_time_period) : 7, // in days } } From b3608a96c5124dff58f25da7f5958aabe4677ea9 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:53:00 +0530 Subject: [PATCH 299/311] feat: #OBS-I321 : Added default query time period in validation schema --- .../src/controllers/DatasetMetrics/DatasetMetrics.json | 4 ++++ .../controllers/DatasetMetrics/DatasetMetricsController.ts | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json index c074544c..3238d1b3 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json @@ -49,6 +49,10 @@ "volume_by_days": { "type": "integer", "minimum": 1 + }, + "query_time_period":{ + "type": "integer", + "minimum": 1 } }, "required": [ diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 1af73eea..650cd68f 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -14,12 +14,12 @@ const datasetMetrics = async (req: Request, res: Response) => { const msgid = _.get(req, "body.params.msgid"); const requestBody = req.body; const dataset_id = _.get(req, "body.request.dataset_id"); - const startDateValue = _.get(req, "body.request.start_date") || config?.data_observability?.default_query_time_period; + const startDateValue = _.get(req, "body.request.query_time_period") || config?.data_observability?.default_query_time_period; const { category, volume_by_days = 10 }: any = req.body.request; const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; - const endDate = dayjs().format(dateFormat); + const endDate = dayjs().add(1, 'day').format(dateFormat); const startDate = dayjs(endDate).subtract(startDateValue, 'day').format(dateFormat); const intervals = `${startDate}/${endDate}`; const isValidSchema = schemaValidation(requestBody, validationSchema); From a46f03a1a3f8bdc2114440e2eab53870eb50a5c0 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 12:59:11 +0530 Subject: [PATCH 300/311] feat: #OBS-I321 : Renamed variable --- .../controllers/DatasetMetrics/DatasetMetricsController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 650cd68f..41d7a051 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -14,13 +14,13 @@ const datasetMetrics = async (req: Request, res: Response) => { const msgid = _.get(req, "body.params.msgid"); const requestBody = req.body; const dataset_id = _.get(req, "body.request.dataset_id"); - const startDateValue = _.get(req, "body.request.query_time_period") || config?.data_observability?.default_query_time_period; + const timePeriod = _.get(req, "body.request.query_time_period") || config?.data_observability?.default_query_time_period; const { category, volume_by_days = 10 }: any = req.body.request; const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; const endDate = dayjs().add(1, 'day').format(dateFormat); - const startDate = dayjs(endDate).subtract(startDateValue, 'day').format(dateFormat); + const startDate = dayjs(endDate).subtract(timePeriod, 'day').format(dateFormat); const intervals = `${startDate}/${endDate}`; const isValidSchema = schemaValidation(requestBody, validationSchema); const results = []; From 4a92961e095257c5b5cff6e2ec943d1dc639132b Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 13:10:10 +0530 Subject: [PATCH 301/311] feat: #OBS-I321 : Removed volume_by_days from request schema --- .../src/controllers/DatasetMetrics/DatasetMetrics.json | 4 ---- .../controllers/DatasetMetrics/DatasetMetricsController.ts | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json index 3238d1b3..70f14029 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json @@ -46,10 +46,6 @@ }, "minItems": 1 }, - "volume_by_days": { - "type": "integer", - "minimum": 1 - }, "query_time_period":{ "type": "integer", "minimum": 1 diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 41d7a051..f84d0b43 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -16,7 +16,7 @@ const datasetMetrics = async (req: Request, res: Response) => { const dataset_id = _.get(req, "body.request.dataset_id"); const timePeriod = _.get(req, "body.request.query_time_period") || config?.data_observability?.default_query_time_period; - const { category, volume_by_days = 10 }: any = req.body.request; + const { category }: any = req.body.request; const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; const endDate = dayjs().add(1, 'day').format(dateFormat); @@ -48,7 +48,7 @@ const datasetMetrics = async (req: Request, res: Response) => { } if (!category || category.includes("data_volume")) { - const dataVolumeResult = await getDataVolume(dataset_id, volume_by_days, dateFormat); + const dataVolumeResult = await getDataVolume(dataset_id, timePeriod, dateFormat); results.push(dataVolumeResult); } From 9ad6f1e34370222ff197c990dd9bb7a14d2717e2 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 13:21:03 +0530 Subject: [PATCH 302/311] feat: #OBS-I321 : Updated timeZone to UTC --- .../src/controllers/DatasetMetrics/queries.ts | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/queries.ts b/api-service/src/controllers/DatasetMetrics/queries.ts index 04701907..ae02e1ac 100644 --- a/api-service/src/controllers/DatasetMetrics/queries.ts +++ b/api-service/src/controllers/DatasetMetrics/queries.ts @@ -7,7 +7,7 @@ export const processingTimeQuery = (intervals: string, dataset_id: string) => ({ intervals: intervals, granularity: { type: "all", - timeZone: "Asia/Kolkata" + timeZone: "UTC" }, filter: { type: "and", @@ -49,7 +49,8 @@ export const totalEventsQuery = (intervals: string, dataset_id: string) => ({ matchValue: dataset_id }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { @@ -77,7 +78,8 @@ export const totalFailedEventsQuery = (intervals: string, dataset_id: string) => matchValue: dataset_id }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { @@ -115,7 +117,7 @@ export const generateTimeseriesQuery = (intervals: string, dataset_id: string) = intervals: intervals, granularity: { type: "all", - timeZone: "Asia/Kolkata" + timeZone: "UTC" }, filter: { type: "and", @@ -137,7 +139,7 @@ export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset_ intervals: intervals, granularity: { type: "all", - timeZone: "Asia/Kolkata" + timeZone: "UTC" }, filter: { type: "and", @@ -181,7 +183,8 @@ export const dataLineageSuccessQuery = (intervals: string, dataset_id: string, c ] }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { @@ -209,7 +212,8 @@ export const generateTransformationFailedQuery = (intervals: string, dataset_id: matchValue: dataset_id }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { @@ -258,7 +262,8 @@ export const generateDedupFailedQuery = (intervals: string, dataset_id: string) matchValue: dataset_id }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { @@ -307,7 +312,8 @@ export const generateDenormFailedQuery = (intervals: string, dataset_id: string) matchValue: dataset_id }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { @@ -369,7 +375,8 @@ export const generateConnectorQuery = (intervals: string, dataset_id: string) => matchValue: dataset_id }, granularity: { - type: "all" + type: "all", + timeZone: "UTC" }, aggregations: [ { From e29f4149e7842a41f6abcbd65e90264da79c6724 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 15 Jan 2025 15:16:16 +0530 Subject: [PATCH 303/311] feat #OBS-I501 : data analyse PII API added --- api-service/src/configs/Config.ts | 2 +- .../connections/commandServiceConnection.ts | 18 +++++++++---- .../ConnectorRegisterController.ts | 5 ++-- .../DataAnalyzePIIController.ts | 26 +++++++++++++++++++ .../controllers/DatasetReset/DatasetReset.ts | 2 +- .../DatasetStatusTransition.ts | 17 ++++++------ .../src/middlewares/userPermissions.json | 12 ++++++--- api-service/src/routes/Router.ts | 2 ++ api-service/src/services/DatasetService.ts | 4 +-- 9 files changed, 65 insertions(+), 23 deletions(-) create mode 100644 api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 702cc13f..26b9d080 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -98,7 +98,7 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), - "path": process.env.command_service_path || "/system/v1/dataset/command" + "paths": JSON.parse(process.env.command_service_paths || '{"dataset":"/system/v1/dataset/command","connector":"/connector/v1/register","analyzePII":"/system/data/v1/analyze/pii"}') }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index fc77fc77..c6f1b411 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -5,11 +5,14 @@ import { v4 } from "uuid"; const commandHost = _.get(config, ["command_service_config", "host"]) const commandPort = _.get(config, ["command_service_config", "port"]) -const commandPath = _.get(config, ["command_service_config", "path"]) +const commandPaths = _.get(config, ["command_service_config", "paths"]) +const datasetPath = _.get(commandPaths, ["dataset"]) +const connectorRegisterPath = _.get(commandPaths, ["connector"]) +const analyzePIIPath = _.get(commandPaths, ["analyzePII"]) export const commandHttpService = axios.create({ baseURL: `${commandHost}:${commandPort}`, headers: { "Content-Type": "application/json" } }); -export const executeCommand = async (id: string, command: string) => { +export const executeCommand = async (id: string, command: string, userToken: string) => { const payload = { "id": v4(), "data": { @@ -17,9 +20,14 @@ export const executeCommand = async (id: string, command: string) => { "command": command } } - return commandHttpService.post(commandPath, payload) + return commandHttpService.post(datasetPath, payload, { headers: { Authorization: userToken }}) } -export const registerConnector = async (requestBody: any) => { - return commandHttpService.post("/connector/v1/register", requestBody) +export const registerConnector = async (requestBody: any, userToken: string) => { + return commandHttpService.post(connectorRegisterPath, requestBody, { headers: { Authorization: userToken }}) +} + +export const detectPII = async (requestBody: any, userToken: string) => { + console.log(`analyzePIIPath : ${analyzePIIPath}`) + return commandHttpService.post(analyzePIIPath, requestBody, { headers: { Authorization: userToken }}) } \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts index 4844ac7e..354577b6 100644 --- a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -22,7 +22,8 @@ const connectorRegisterController = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await registerConnector(payload); + const userToken = req.get('authorization') as string; + const registryResponse = await registerConnector(payload, userToken); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { @@ -118,4 +119,4 @@ const extractFileNameFromPath = (filePath: string): string[] => { return filePath.match(regex) || []; }; -export default connectorRegisterController; +export default connectorRegisterController; \ No newline at end of file diff --git a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts new file mode 100644 index 00000000..50251e91 --- /dev/null +++ b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts @@ -0,0 +1,26 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import * as _ from "lodash"; +import logger from "../../logger"; +import { detectPII } from "../../connections/commandServiceConnection"; + +const code = "FAILED_TO_DETECT_PII"; +export const dataAnalyzePII = async (req: Request, res: Response) => { + const apiId = _.get(req, 'id') + try { + const userToken = req.get('authorization') as string; + const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'data']), userToken); + logger.info({apiId , message: `Detected PII successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: piiSuggestionsResponse?.data}) + } catch (error: any) { + const errMessage = _.get(error, "response.data.detail") + logger.error(error, apiId, code); + let errorMessage = error; + const statusCode = _.get(error, "status") + if (!statusCode || statusCode == 500) { + errorMessage = { code, message: errMessage || "Failed to detect pii" } + } + ResponseHandler.errorResponse(errorMessage, req, res); + } +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index cc5aef3c..212f8bc9 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -34,7 +34,7 @@ const datasetReset = async (req: Request, res: Response) => { if (category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() if (pipeLineStatus == HealthStatus.UnHealthy) { - await restartPipeline({ "dataset": { "dataset_id": datasetId } }) + await restartPipeline({ "dataset": { "dataset_id": datasetId } }, userToken) } } else if (category == "query") { const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index ff1502d7..e09fa5b8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -50,6 +50,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); + const userToken = req.get('authorization') as string; validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) @@ -64,10 +65,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userID); + await publishDataset(dataset, userID, userToken); break; case "Retire": - await retireDataset(dataset, userID); + await retireDataset(dataset, userID, userToken); break; default: throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) @@ -134,7 +135,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userID: any) => { +const publishDataset = async (dataset: Record, userID: any, userToken: string) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record validateStorageSupport(draftDataset); @@ -142,7 +143,7 @@ const publishDataset = async (dataset: Record, userID: any) => { _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) - await datasetService.publishDataset(draftDataset) + await datasetService.publishDataset(draftDataset, userToken); } const validateAndUpdateDenormConfig = async (draftDataset: Record) => { @@ -227,11 +228,11 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record, updated_by: any) => { +const retireDataset = async (dataset: Record, updated_by: any, userToken: string) => { await canRetireIfMasterDataset(dataset); await datasetService.retireDataset(dataset, updated_by); - await restartPipeline(dataset); + await restartPipeline(dataset, userToken); } @@ -254,8 +255,8 @@ const canRetireIfMasterDataset = async (dataset: Record) => { } } -export const restartPipeline = async (dataset: Record) => { - return executeCommand(dataset.id, "RESTART_PIPELINE") +export const restartPipeline = async (dataset: Record, userToken: string) => { + return executeCommand(dataset.id, "RESTART_PIPELINE", userToken) } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index 88393ba8..1e4aca1c 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -51,7 +51,8 @@ "api.datasets.dataschema" ], "data": [ - "api.data.in" + "api.data.in", + "api.data.analyze.pii" ], "queryTemplate": [ "api.query.template.create", @@ -93,7 +94,8 @@ "schema", "file", "connector", - "sqlQuery" + "sqlQuery", + "data" ], "dataset_manager": [ "general_access", @@ -105,7 +107,8 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "data" ], "admin": [ "general_access", @@ -121,7 +124,8 @@ "alert", "metric", "silence", - "notificationChannel" + "notificationChannel", + "data" ], "operations_admin": [ "alert", diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 60930f05..f9cc7905 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -33,6 +33,7 @@ import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; import dataMetrics from "../controllers/DataMetrics/DataMetricsController"; import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; +import { dataAnalyzePII } from "../controllers/DataAnalyzePII/DataAnalyzePIIController"; export const router = express.Router(); @@ -61,6 +62,7 @@ router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), o router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); +router.post("/data/analyze/pii", setDataToRequestObject("api.data.analyze.pii"), onRequest({ entity: Entity.Management }),checkRBAC.handler(), dataAnalyzePII); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), dataMetrics) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index a5821892..b953b47f 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -303,7 +303,7 @@ class DatasetService { } } - publishDataset = async (draftDataset: Record) => { + publishDataset = async (draftDataset: Record, userToken: string) => { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() @@ -326,7 +326,7 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET", userToken); } From 3ddf7c758414f7a81d2782fcbe8c72a351af5e69 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 15 Jan 2025 15:25:34 +0530 Subject: [PATCH 304/311] feat #OBS-I501 user token fix --- api-service/src/controllers/DatasetReset/DatasetReset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index 212f8bc9..892db86b 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -29,7 +29,7 @@ const datasetReset = async (req: Request, res: Response) => { const category = _.get(req, ["body", "request", "category"]); const datasetId = _.get(req, ["params"," dataset_id"]); - + const userToken = req.get('authorization') as string; await validateRequest(req); if (category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() From 867afdfb476e31ad0e248f70296ceeb1fff490a7 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 15:51:01 +0530 Subject: [PATCH 305/311] feat : #OBS-I321 : added swagger doc --- api-service/swagger-doc/openapi_v2.yml | 209 +++++++++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 564ab32c..f379c86a 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -10096,3 +10096,212 @@ paths: description: Successful response content: application/json: {} + /v2/dataset/metrics: + post: + tags: + - Dataset metrics + summary: Dataset metrics + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.dataset.metrics + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: test + category: + - data_freshness + - data_observability + - data_volume + - data_lineage + - connectors + - data_quality + query_time_period: 1 + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Get all the data' + value: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:24:21+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: ef86ff9c-4342-4cdc-a573-d469cbf436eb + responseCode: OK + result: + - category: data_freshness + status: Healthy + components: + - type: average_time_difference_in_min + threshold: 5 + value: 0 + status: Healthy + - type: freshness_query_time_in_min + threshold: 10 + value: 0 + status: Healthy + - category: data_observability + status: Unhealthy + components: + - type: data_observability_health + status: Unhealthy + - type: failure_percentage + value: 35.39823008849557 + - type: threshold_percentage + value: 5 + - type: importance_score + value: 0 + - category: data_volume + components: + - type: events_per_hour + value: 0 + - type: events_per_day + value: 95 + - type: events_per_n_day + value: 765 + - type: volume_percentage_by_hour + value: 0 + - type: volume_percentage_by_day + value: -87.58169934640523 + - type: volume_percentage_by_week + value: 0 + - type: growth_rate_percentage + value: 0 + - category: data_lineage + components: + - type: transformation_success + value: 765 + - type: dedup_success + value: 384 + - type: denormalization_success + value: 572 + - type: total_success + value: 1339 + - type: total_failed + value: 1034 + - type: transformation_failed + value: 1 + - type: dedup_failed + value: 574 + - type: denorm_failed + value: 193 + - category: connectors + components: + category: connectors + components: + - id: failed + type: failed + value: 1608 + - id: api + type: success + value: 765 + - category: data_quality + components: + - type: incidents_failed + value: 1034 + - type: incidents_success + value: 1339 + - type: total_incidents + value: 2373 + example-1: + summary: 'Success : Get particular category data' + value: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:25:13+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 10915ec3-a265-4502-bc9b-673562f94512 + responseCode: OK + result: + - category: data_lineage + components: + - type: transformation_success + value: 765 + - type: dedup_success + value: 384 + - type: denormalization_success + value: 572 + - type: total_success + value: 1339 + - type: total_failed + value: 1034 + - type: transformation_failed + value: 1 + - type: dedup_failed + value: 574 + - type: denorm_failed + value: 193 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:23:22+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a6a26834-bb32-4e32-8dfd-473b0d21c8e8 + responseCode: BAD_REQUEST + error: + code: DATA_OUT_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:15:30+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 25107bba-6d88-4e9b-b951-61e29a72b97a + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: Dataset with id test not found in live table + trace: '' + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:29:49+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 211553f9-ad91-4a4d-a6ba-d77d3e14b7ac + responseCode: FAILED + error: + code: FAILED_TO_FETCH_METRICS + message: Error while fetching metrics + trace: '' \ No newline at end of file From 821d8b4ad463acfb696adc8bf723ff6a2eaeaf19 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 15:54:18 +0530 Subject: [PATCH 306/311] feat: #OBS-I321 : added postman collection --- .../Obsrv v2 apis.postman_collection.json | 288 +++++++++++++++++- 1 file changed, 286 insertions(+), 2 deletions(-) diff --git a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json index 041ee138..d813420c 100644 --- a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -4,8 +4,8 @@ "name": "Obsrv v2 apis", "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "26192103", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" + "_exporter_id": "33916975", + "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=33916975" }, "item": [ { @@ -6654,6 +6654,290 @@ "response": [] } ] + }, + { + "name": "Dataset metrics", + "item": [ + { + "name": "Dataset metrics", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/dataset/metrics" + }, + "response": [ + { + "name": "Failure: Dataset not found", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/dataset/metrics" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "330" + }, + { + "key": "ETag", + "value": "W/\"14a-RKjp44K2gCKCdq3dClRy/aYxcd8\"" + }, + { + "key": "Date", + "value": "Wed, 15 Jan 2025 09:45:30 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:15:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"25107bba-6d88-4e9b-b951-61e29a72b97a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id test not found in live table\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Request validation failed", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/dataset/metrics" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "362" + }, + { + "key": "ETag", + "value": "W/\"16a-pmYCarF3FSxd3J7zVBlhwST+rq8\"" + }, + { + "key": "Date", + "value": "Wed, 15 Jan 2025 09:53:22 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:23:22+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a6a26834-bb32-4e32-8dfd-473b0d21c8e8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: Get all the data", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/dataset/metrics" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1780" + }, + { + "key": "ETag", + "value": "W/\"6f4-nUo39jrIJBlBgZ4cY4UL84W7UNU\"" + }, + { + "key": "Date", + "value": "Wed, 15 Jan 2025 09:54:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:24:21+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ef86ff9c-4342-4cdc-a573-d469cbf436eb\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"category\": \"data_freshness\",\n \"status\": \"Healthy\",\n \"components\": [\n {\n \"type\": \"average_time_difference_in_min\",\n \"threshold\": 5,\n \"value\": 0,\n \"status\": \"Healthy\"\n },\n {\n \"type\": \"freshness_query_time_in_min\",\n \"threshold\": 10,\n \"value\": 0,\n \"status\": \"Healthy\"\n }\n ]\n },\n {\n \"category\": \"data_observability\",\n \"status\": \"Unhealthy\",\n \"components\": [\n {\n \"type\": \"data_observability_health\",\n \"status\": \"Unhealthy\"\n },\n {\n \"type\": \"failure_percentage\",\n \"value\": 35.39823008849557\n },\n {\n \"type\": \"threshold_percentage\",\n \"value\": 5\n },\n {\n \"type\": \"importance_score\",\n \"value\": 0\n }\n ]\n },\n {\n \"category\": \"data_volume\",\n \"components\": [\n {\n \"type\": \"events_per_hour\",\n \"value\": 0\n },\n {\n \"type\": \"events_per_day\",\n \"value\": 95\n },\n {\n \"type\": \"events_per_n_day\",\n \"value\": 765\n },\n {\n \"type\": \"volume_percentage_by_hour\",\n \"value\": 0\n },\n {\n \"type\": \"volume_percentage_by_day\",\n \"value\": -87.58169934640523\n },\n {\n \"type\": \"volume_percentage_by_week\",\n \"value\": 0\n },\n {\n \"type\": \"growth_rate_percentage\",\n \"value\": 0\n }\n ]\n },\n {\n \"category\": \"data_lineage\",\n \"components\": [\n {\n \"type\": \"transformation_success\",\n \"value\": 765\n },\n {\n \"type\": \"dedup_success\",\n \"value\": 384\n },\n {\n \"type\": \"denormalization_success\",\n \"value\": 572\n },\n {\n \"type\": \"total_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"transformation_failed\",\n \"value\": 1\n },\n {\n \"type\": \"dedup_failed\",\n \"value\": 574\n },\n {\n \"type\": \"denorm_failed\",\n \"value\": 193\n }\n ]\n },\n {\n \"category\": \"connectors\",\n \"components\": {\n \"category\": \"connectors\",\n \"components\": [\n {\n \"id\": \"failed\",\n \"type\": \"failed\",\n \"value\": 1608\n },\n {\n \"id\": \"api\",\n \"type\": \"success\",\n \"value\": 765\n }\n ]\n }\n },\n {\n \"category\": \"data_quality\",\n \"components\": [\n {\n \"type\": \"incidents_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"incidents_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_incidents\",\n \"value\": 2373\n }\n ]\n }\n ]\n}" + }, + { + "name": "Success : Get particular category data", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n // \"data_freshness\",\n // \"data_observability\",\n // \"data_volume\",\n \"data_lineage\"\n // \"connectors\",\n // \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/dataset/metrics" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "594" + }, + { + "key": "ETag", + "value": "W/\"252-mKqBPcpjk0LPbqNHyghckQWQnHs\"" + }, + { + "key": "Date", + "value": "Wed, 15 Jan 2025 09:55:13 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:25:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"10915ec3-a265-4502-bc9b-673562f94512\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"category\": \"data_lineage\",\n \"components\": [\n {\n \"type\": \"transformation_success\",\n \"value\": 765\n },\n {\n \"type\": \"dedup_success\",\n \"value\": 384\n },\n {\n \"type\": \"denormalization_success\",\n \"value\": 572\n },\n {\n \"type\": \"total_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"transformation_failed\",\n \"value\": 1\n },\n {\n \"type\": \"dedup_failed\",\n \"value\": 574\n },\n {\n \"type\": \"denorm_failed\",\n \"value\": 193\n }\n ]\n }\n ]\n}" + }, + { + "name": "Failure: Failed to fetch metrics", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/dataset/metrics" + }, + "status": "Internal Server Error", + "code": 500, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "317" + }, + { + "key": "ETag", + "value": "W/\"13d-xfZKi6jq4UY4l8MNfhQmydAroPc\"" + }, + { + "key": "Date", + "value": "Wed, 15 Jan 2025 09:59:49 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:29:49+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"211553f9-ad91-4a4d-a6ba-d77d3e14b7ac\"\n },\n \"responseCode\": \"FAILED\",\n \"error\": {\n \"code\": \"FAILED_TO_FETCH_METRICS\",\n \"message\": \"Error while fetching metrics\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] } ] } \ No newline at end of file From 42fb4cc8302d455426facf3a17a6d521e0b4f16c Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 16:19:54 +0530 Subject: [PATCH 307/311] feat : #OBS-I321 : return result without modifying --- api-service/src/services/DatasetMetricsService.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api-service/src/services/DatasetMetricsService.ts b/api-service/src/services/DatasetMetricsService.ts index 2c879d64..d22bee3c 100644 --- a/api-service/src/services/DatasetMetricsService.ts +++ b/api-service/src/services/DatasetMetricsService.ts @@ -209,10 +209,7 @@ export const getConnectors = async (dataset_id: string, intervals: string) => { })) }; - return { - category: "connectors", - components: result - }; + return result; }; export const getDataQuality = async (dataset_id: string, intervals: string) => { From 380f4bbbcb7f737b2eed647dd730865e1e63a6e1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 15 Jan 2025 16:31:14 +0530 Subject: [PATCH 308/311] feat : #OBS-I321 : updated doc --- api-service/swagger-doc/openapi_v2.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index f379c86a..8a65e8e0 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -7737,6 +7737,13 @@ paths: message: Entry topic is not defined trace: '' /v2/data/query/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset post: tags: - Data Query @@ -10200,14 +10207,12 @@ paths: value: 193 - category: connectors components: - category: connectors - components: - - id: failed - type: failed - value: 1608 - - id: api - type: success - value: 765 + - id: failed + type: failed + value: 1608 + - id: api + type: success + value: 765 - category: data_quality components: - type: incidents_failed From 45c6e3423aebcac69e82fb74c5ebd35710e7af29 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 15 Jan 2025 16:58:24 +0530 Subject: [PATCH 309/311] feat #OBS-I501 request response changes --- api-service/src/connections/commandServiceConnection.ts | 1 - .../controllers/DataAnalyzePII/DataAnalyzePIIController.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index c6f1b411..d07b01b2 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -28,6 +28,5 @@ export const registerConnector = async (requestBody: any, userToken: string) => } export const detectPII = async (requestBody: any, userToken: string) => { - console.log(`analyzePIIPath : ${analyzePIIPath}`) return commandHttpService.post(analyzePIIPath, requestBody, { headers: { Authorization: userToken }}) } \ No newline at end of file diff --git a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts index 50251e91..7e4a3af8 100644 --- a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts +++ b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts @@ -10,9 +10,9 @@ export const dataAnalyzePII = async (req: Request, res: Response) => { const apiId = _.get(req, 'id') try { const userToken = req.get('authorization') as string; - const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'data']), userToken); + const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'request']), userToken); logger.info({apiId , message: `Detected PII successfully` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: piiSuggestionsResponse?.data}) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: _.get(piiSuggestionsResponse, ["data", "result"]) }); } catch (error: any) { const errMessage = _.get(error, "response.data.detail") logger.error(error, apiId, code); From 4312c79566d7211a3337559738f842f48ee88fa4 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Thu, 16 Jan 2025 11:31:51 +0530 Subject: [PATCH 310/311] doc #OBS-I501 : add postman collection --- .../Obsrv v2 apis.postman_collection.json | 2051 +++++++++++++++-- 1 file changed, 1883 insertions(+), 168 deletions(-) diff --git a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json index d813420c..cd8d4689 100644 --- a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -1,11 +1,11 @@ { "info": { - "_postman_id": "d3ffc748-3d71-4395-9298-e065845f7bfb", + "_postman_id": "b94beefb-2a07-49ac-bd5b-07d66fe7ba85", "name": "Obsrv v2 apis", "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", - "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "33916975", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=33916975" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "406673", + "_collection_link": "https://galactic-flare-389256.postman.co/workspace/config-service-workspace~0ba551f8-be4b-4755-9d4d-bf4578113e5f/collection/406673-b94beefb-2a07-49ac-bd5b-07d66fe7ba85?action=share&source=collection_link&creator=406673" }, "item": [ { @@ -30,7 +30,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create", + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + }, "description": "This API allows you to create new datasets used by the analytical data source." }, "response": [ @@ -53,7 +64,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -110,7 +132,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -167,7 +200,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -224,7 +268,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -281,7 +336,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Bad Request", "code": 400, @@ -338,7 +404,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -395,7 +472,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -454,7 +542,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url", + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + }, "description": "This API generates presigned URLs to upload or download files from cloud" }, "response": [ @@ -477,7 +576,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "OK", "code": 200, @@ -535,7 +645,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "OK", "code": 200, @@ -593,7 +714,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "Bad Request", "code": 400, @@ -651,7 +783,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "Bad Request", "code": 400, @@ -710,7 +853,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update", + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + }, "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors" }, "response": [ @@ -733,7 +887,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "OK", "code": 200, @@ -790,7 +955,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "OK", "code": 200, @@ -847,7 +1023,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Conflict", "code": 409, @@ -904,7 +1091,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Not Found", "code": 404, @@ -961,7 +1159,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Bad Request", "code": 400, @@ -1018,7 +1227,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Bad Request", "code": 400, @@ -1100,7 +1320,19 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3000/v2/datasets/read/master-test" + "url": { + "raw": "localhost:3000/v2/datasets/read/master-test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "read", + "master-test" + ] + } }, "status": "OK", "code": 200, @@ -1482,7 +1714,19 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3000/v2/datasets/read/new_telemetry_record.1" + "url": { + "raw": "localhost:3000/v2/datasets/read/new_telemetry_record.1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "read", + "new_telemetry_record.1" + ] + } }, "status": "Not Found", "code": 404, @@ -1536,7 +1780,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list", + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + }, "description": "This API allows you to list all datasets. User can apply filters on dataset status and type." }, "response": [ @@ -1554,7 +1809,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1606,7 +1872,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1658,15 +1935,26 @@ } } }, - "url": "localhost:3000/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" }, { "key": "Content-Type", @@ -1710,7 +1998,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1762,7 +2061,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "Bad Request", "code": 400, @@ -1825,7 +2135,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema", + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + }, "description": "This api is used to generate data schema for the given dataset event." }, "response": [ @@ -1852,7 +2173,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "OK", "code": 200, @@ -1913,7 +2245,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "Bad Request", "code": 400, @@ -1974,7 +2317,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "Bad Request", "code": 400, @@ -2028,7 +2382,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition", + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + }, "description": "This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset." }, "response": [ @@ -2046,7 +2411,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, @@ -2069,7 +2445,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, @@ -2092,7 +2479,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, @@ -2115,7 +2513,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, @@ -2138,7 +2547,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2161,7 +2581,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, @@ -2184,7 +2615,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2207,7 +2649,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2230,7 +2683,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, @@ -2253,7 +2717,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, @@ -2276,7 +2751,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2299,7 +2785,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2322,7 +2819,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2780,10 +3288,10 @@ ] } }, - "_postman_previewlanguage": null, - "header": null, + "_postman_previewlanguage": "Text", + "header": [], "cookie": [], - "body": null + "body": "" }, { "name": "Dataset import", @@ -2823,10 +3331,10 @@ ] } }, - "_postman_previewlanguage": null, - "header": null, + "_postman_previewlanguage": "Text", + "header": [], "cookie": [], - "body": null + "body": "" } ] }, @@ -3121,7 +3629,18 @@ } } }, - "url": "localhost:3000/v2/datasets/copy" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, "response": [ { @@ -3143,7 +3662,18 @@ } } }, - "url": "localhost:3000/v2/datasets/copy" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, "status": "OK", "code": 200, @@ -3200,7 +3730,18 @@ } } }, - "url": "localhost:3000/v2/datasets/copy" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, "status": "Not Found", "code": 404, @@ -3260,13 +3801,25 @@ } } }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "response": [ - { - "name": "Failure: Invalid request body, filter option array should not be empty", - "originalRequest": { - "method": "POST", + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } + }, + "response": [ + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", "header": [], "body": { "mode": "raw", @@ -3277,7 +3830,19 @@ } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "Bad Request", "code": 400, @@ -3329,7 +3894,19 @@ } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3381,7 +3958,19 @@ } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3433,7 +4022,19 @@ } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3485,7 +4086,19 @@ } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3530,7 +4143,20 @@ "request": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ] + } }, "response": [ { @@ -3538,7 +4164,20 @@ "originalRequest": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ] + } }, "status": "OK", "code": 200, @@ -3643,7 +4282,20 @@ "originalRequest": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-conn", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-conn" + ] + } }, "status": "Not Found", "code": 404, @@ -3707,7 +4359,19 @@ } } }, - "url": "localhost:3000/v2/data/in/sb-telemetry-imported" + "url": { + "raw": "localhost:3000/v2/data/in/sb-telemetry-imported", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sb-telemetry-imported" + ] + } }, "response": [ { @@ -3730,7 +4394,19 @@ } } }, - "url": "localhost:3000/v2/data/in/sample-test" + "url": { + "raw": "localhost:3000/v2/data/in/sample-test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sample-test" + ] + } }, "status": "Not Found", "code": 404, @@ -3788,7 +4464,19 @@ } } }, - "url": "localhost:3000/v2/data/in/test2" + "url": { + "raw": "localhost:3000/v2/data/in/test2", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test2" + ] + } }, "status": "Bad Request", "code": 400, @@ -3846,7 +4534,19 @@ } } }, - "url": "localhost:3000/v2/data/in/test" + "url": { + "raw": "localhost:3000/v2/data/in/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test" + ] + } }, "status": "OK", "code": 200, @@ -3904,7 +4604,19 @@ } } }, - "url": "localhost:3000/v2/data/in/test2" + "url": { + "raw": "localhost:3000/v2/data/in/test2", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test2" + ] + } }, "status": "OK", "code": 200, @@ -3962,7 +4674,19 @@ } } }, - "url": "localhost:3000/v2/data/in/added-tags" + "url": { + "raw": "localhost:3000/v2/data/in/added-tags", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "added-tags" + ] + } }, "status": "Not Found", "code": 404, @@ -4020,7 +4744,19 @@ } } }, - "url": "localhost:3000/v2/data/in/sample1" + "url": { + "raw": "localhost:3000/v2/data/in/sample1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sample1" + ] + } }, "status": "Not Found", "code": 404, @@ -4085,7 +4821,19 @@ } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "response": [ { @@ -4109,7 +4857,19 @@ } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4168,7 +4928,19 @@ } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4227,7 +4999,19 @@ } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4286,7 +5070,19 @@ } } }, - "url": "localhost:3000/v2/data/query/telemetry-eventssss" + "url": { + "raw": "localhost:3000/v2/data/query/telemetry-eventssss", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "telemetry-eventssss" + ] + } }, "status": "Not Found", "code": 404, @@ -4345,7 +5141,19 @@ } } }, - "url": "localhost:3000/v2/data/query/telemetry-events" + "url": { + "raw": "localhost:3000/v2/data/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "telemetry-events" + ] + } }, "status": "Bad Request", "code": 400, @@ -4404,7 +5212,19 @@ } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "Not Found", "code": 404, @@ -4463,7 +5283,19 @@ } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4522,13 +5354,25 @@ } } }, - "url": "localhost:3000/v2/data/query/test" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { "key": "X-Powered-By", "value": "Express" }, @@ -4581,7 +5425,19 @@ } } }, - "url": "localhost:3000/v2/data/query/undefined" + "url": { + "raw": "localhost:3000/v2/data/query/undefined", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "undefined" + ] + } }, "status": "Not Found", "code": 404, @@ -4671,7 +5527,19 @@ "request": { "method": "GET", "header": [], - "url": "localhost:3000/v2/template/read/sql_template_11" + "url": { + "raw": "localhost:3000/v2/template/read/sql_template_11", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql_template_11" + ] + } }, "response": [ { @@ -4679,7 +5547,19 @@ "originalRequest": { "method": "GET", "header": [], - "url": "localhost:3000/v2/template/read/jsontemplate111" + "url": { + "raw": "localhost:3000/v2/template/read/jsontemplate111", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "jsontemplate111" + ] + } }, "status": "OK", "code": 200, @@ -4722,7 +5602,19 @@ "originalRequest": { "method": "GET", "header": [], - "url": "localhost:3000/v2/template/read/sql1" + "url": { + "raw": "localhost:3000/v2/template/read/sql1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql1" + ] + } }, "status": "OK", "code": 200, @@ -4765,7 +5657,19 @@ "originalRequest": { "method": "GET", "header": [], - "url": "localhost:3000/v2/template/read/sql100" + "url": { + "raw": "localhost:3000/v2/template/read/sql100", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql100" + ] + } }, "status": "Not Found", "code": 404, @@ -4810,7 +5714,19 @@ "request": { "method": "DELETE", "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "url": { + "raw": "localhost:3000/v2/template/delete/yashash-k", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "yashash-k" + ] + } }, "response": [ { @@ -4818,7 +5734,19 @@ "originalRequest": { "method": "DELETE", "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "url": { + "raw": "localhost:3000/v2/template/delete/yashash-k", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "yashash-k" + ] + } }, "status": "OK", "code": 200, @@ -4861,7 +5789,19 @@ "originalRequest": { "method": "DELETE", "header": [], - "url": "localhost:3000/v2/template/delete/json_template" + "url": { + "raw": "localhost:3000/v2/template/delete/json_template", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "json_template" + ] + } }, "status": "Not Found", "code": 404, @@ -4921,7 +5861,18 @@ } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, "response": [ { @@ -4944,7 +5895,18 @@ } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, "status": "OK", "code": 200, @@ -5002,7 +5964,18 @@ } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, "status": "OK", "code": 200, @@ -5060,7 +6033,18 @@ } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -5118,7 +6102,18 @@ } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, "status": "Bad Request", "code": 400, @@ -5176,7 +6171,18 @@ } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, "status": "Bad Request", "code": 400, @@ -5230,7 +6236,18 @@ } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } }, "response": [ { @@ -5247,7 +6264,18 @@ } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } }, "status": "OK", "code": 200, @@ -5299,7 +6327,18 @@ } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } }, "status": "OK", "code": 200, @@ -5351,7 +6390,18 @@ } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } }, "status": "OK", "code": 200, @@ -5403,7 +6453,18 @@ } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } }, "status": "OK", "code": 200, @@ -5457,7 +6518,19 @@ } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } }, "response": [ { @@ -5474,7 +6547,19 @@ } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } }, "status": "OK", "code": 200, @@ -5526,7 +6611,19 @@ } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } }, "status": "Bad Request", "code": 400, @@ -5578,7 +6675,19 @@ } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } }, "status": "Bad Request", "code": 400, @@ -5630,7 +6739,19 @@ } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } }, "status": "Bad Request", "code": 400, @@ -5682,8 +6803,20 @@ } } }, - "url": "localhost:3000/v2/template/update/sql11template1" - }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", @@ -5736,7 +6869,19 @@ } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } }, "response": [ { @@ -5830,7 +6975,19 @@ } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } }, "status": "Not Found", "code": 404, @@ -5882,7 +7039,19 @@ } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } }, "status": "Not Found", "code": 404, @@ -5934,7 +7103,19 @@ } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } }, "status": "Bad Request", "code": 400, @@ -5992,7 +7173,19 @@ } } }, - "url": "localhost:3000/v2/template/query/jsontemplate1" + "url": { + "raw": "localhost:3000/v2/template/query/jsontemplate1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "jsontemplate1" + ] + } }, "status": "OK", "code": 200, @@ -6044,7 +7237,19 @@ } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } }, "status": "OK", "code": 200, @@ -6103,7 +7308,19 @@ } } }, - "url": "localhost:3000/data/v2/schema/validate" + "url": { + "raw": "localhost:3000/data/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "data", + "v2", + "schema", + "validate" + ] + } }, "response": [ { @@ -6120,7 +7337,18 @@ } } }, - "url": "localhost:3000/v2/schema/validate" + "url": { + "raw": "localhost:3000/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "schema", + "validate" + ] + } }, "status": "OK", "code": 200, @@ -6172,7 +7400,18 @@ } } }, - "url": "localhost:3000/v2/schema/validate" + "url": { + "raw": "localhost:3000/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "schema", + "validate" + ] + } }, "status": "Not Found", "code": 404, @@ -6222,7 +7461,18 @@ "request": { "method": "POST", "header": [], - "url": "{{HOST_IP}}/alerts/v1/notifications/search" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "search" + ] + } }, "response": [] }, @@ -6240,7 +7490,18 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/notifications/create" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "create" + ] + } }, "response": [] }, @@ -6249,7 +7510,19 @@ "request": { "method": "GET", "header": [], - "url": "{{HOST_IP}}/alerts/v1/notifications/publish/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/publish/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "publish", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } }, "response": [] }, @@ -6267,7 +7540,18 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/notifications/test" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/test", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "test" + ] + } }, "response": [] }, @@ -6285,7 +7569,19 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/notifications/update/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/update/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "update", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } }, "response": [] }, @@ -6294,7 +7590,19 @@ "request": { "method": "DELETE", "header": [], - "url": "{{HOST_IP}}/alerts/v1/notifications/delete/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/delete/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "delete", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } }, "response": [] }, @@ -6312,7 +7620,18 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/notifications/create" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "create" + ] + } }, "response": [] } @@ -6335,7 +7654,18 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/silence/create" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "create" + ] + } }, "response": [] }, @@ -6344,7 +7674,19 @@ "request": { "method": "GET", "header": [], - "url": "{{HOST_IP}}/alerts/v1/silence/get/0ec52a4c-4529-4f70-b6f1-0de4b1e81c13" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/get/0ec52a4c-4529-4f70-b6f1-0de4b1e81c13", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "get", + "0ec52a4c-4529-4f70-b6f1-0de4b1e81c13" + ] + } }, "response": [] }, @@ -6353,7 +7695,19 @@ "request": { "method": "DELETE", "header": [], - "url": "{{HOST_IP}}/alerts/v1/silence/delete/39c7eec1-db78-4f5d-8e77-38385cc3b7dc" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/delete/39c7eec1-db78-4f5d-8e77-38385cc3b7dc", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "delete", + "39c7eec1-db78-4f5d-8e77-38385cc3b7dc" + ] + } }, "response": [] }, @@ -6362,7 +7716,18 @@ "request": { "method": "GET", "header": [], - "url": "{{HOST_IP}}/alerts/v1/silence/search" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "search" + ] + } }, "response": [] }, @@ -6380,7 +7745,19 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/silence/update/da46091b-65db-4df9-85d1-c2341168bcb8" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/update/da46091b-65db-4df9-85d1-c2341168bcb8", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "update", + "da46091b-65db-4df9-85d1-c2341168bcb8" + ] + } }, "response": [] } @@ -6394,7 +7771,18 @@ "request": { "method": "DELETE", "header": [], - "url": "{{HOST_IP}}/alerts/v1/delete/0c3e08d4-8e4f-4810-b591-f5a294ee378e", + "url": { + "raw": "{{HOST_IP}}/alerts/v1/delete/0c3e08d4-8e4f-4810-b591-f5a294ee378e", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "delete", + "0c3e08d4-8e4f-4810-b591-f5a294ee378e" + ] + }, "description": "This URLwill provided access to user to delete any custom rule." }, "response": [] @@ -6413,7 +7801,18 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/update/a6c46c48-79c1-4e34-9552-fe8f13d3c73e", + "url": { + "raw": "{{HOST_IP}}/alerts/v1/update/a6c46c48-79c1-4e34-9552-fe8f13d3c73e", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "update", + "a6c46c48-79c1-4e34-9552-fe8f13d3c73e" + ] + }, "description": "This URL will provide access to users to edit any properties in the Alert rule and save the changes." }, "response": [] @@ -6430,7 +7829,18 @@ "mode": "raw", "raw": "" }, - "url": "{{HOST_IP}}/alerts/v1/publish/c6c93917-9e5e-4aa6-9986-1c53c45ba7fe" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/publish/c6c93917-9e5e-4aa6-9986-1c53c45ba7fe", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "publish", + "c6c93917-9e5e-4aa6-9986-1c53c45ba7fe" + ] + } }, "response": [] }, @@ -6500,7 +7910,17 @@ "mode": "raw", "raw": "{\n \"name\": \"Total Api Calls1\",\n \"manager\":\"grafana\",\n \"description\": \"This alert is set up to notify you when the CPU usage on a host reaches a Low level.\",\n \"expression\": \"(node_total_api_calls) > 20\",\n \"category\": \"Infra\",\n \"severity\":\"warning\",\n \"frequency\": \"1m\",\n \"interval\": \"1m\",\n \"labels\": {\n \"component\": \"api\",\n \"notificationChannel\": \"slack\"\n },\n \"annotations\":{\n \"summary\":\"Host Low CPU usage\"\n },\n \"metadata\": {\n \"query\": [\n {\n \"refId\": \"A\",\n \"datasourceUid\": \"$datasourceUid\",\n \"queryType\": \"\",\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n },\n \"model\": {\n \"refId\": \"A\",\n \"hide\": false,\n \"editorMode\": \"code\",\n \"expr\": \"node_total_api_calls\",\n \"legendFormat\": \"__auto\",\n \"range\": true\n }\n },\n {\n \"refId\": \"B\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"B\",\n \"hide\": false,\n \"type\": \"reduce\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"B\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"reducer\": \"last\",\n \"expression\": \"A\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n },\n {\n \"refId\": \"C\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"C\",\n \"hide\": false,\n \"type\": \"threshold\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [\n 20\n ],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"C\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"expression\": \"B\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n }\n ]\n }\n}" }, - "url": "{{HOST_IP}}/alerts/v1/create" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "create" + ] + } }, "response": [] }, @@ -6518,7 +7938,17 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/search" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "search" + ] + } }, "response": [] }, @@ -6527,7 +7957,18 @@ "request": { "method": "GET", "header": [], - "url": "{{HOST_IP}}/alerts/v1/get/dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/get/dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "get", + "dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff" + ] + } }, "response": [] } @@ -6550,7 +7991,19 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/metric/alias/create" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "create" + ] + } }, "response": [] }, @@ -6568,7 +8021,19 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/metric/alias/search" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "search" + ] + } }, "response": [] }, @@ -6586,7 +8051,20 @@ } } }, - "url": "{{HOST_IP}}/alerts/v1/metric/alias/update/48aad878-d59c-4046-aeb5-fb87aefbcfed" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/update/48aad878-d59c-4046-aeb5-fb87aefbcfed", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "update", + "48aad878-d59c-4046-aeb5-fb87aefbcfed" + ] + } }, "response": [] }, @@ -6595,7 +8073,20 @@ "request": { "method": "DELETE", "header": [], - "url": "{{HOST_IP}}/alerts/v1/metric/alias/delete/f168707f-5908-4e2e-9111-e73c41e2700d" + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/delete/f168707f-5908-4e2e-9111-e73c41e2700d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "delete", + "f168707f-5908-4e2e-9111-e73c41e2700d" + ] + } }, "response": [] } @@ -6649,7 +8140,18 @@ } ] }, - "url": "localhost:3000/v2/connector/register" + "url": { + "raw": "localhost:3000/v2/connector/register", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connector", + "register" + ] + } }, "response": [] } @@ -6672,7 +8174,18 @@ } } }, - "url": "localhost:3007/v2/dataset/metrics" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, "response": [ { @@ -6689,7 +8202,18 @@ } } }, - "url": "localhost:3007/v2/dataset/metrics" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, "status": "Not Found", "code": 404, @@ -6741,7 +8265,18 @@ } } }, - "url": "localhost:3007/v2/dataset/metrics" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, "status": "Bad Request", "code": 400, @@ -6793,7 +8328,18 @@ } } }, - "url": "localhost:3007/v2/dataset/metrics" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, "status": "OK", "code": 200, @@ -6845,7 +8391,18 @@ } } }, - "url": "localhost:3007/v2/dataset/metrics" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, "status": "OK", "code": 200, @@ -6897,7 +8454,18 @@ } } }, - "url": "localhost:3007/v2/dataset/metrics" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, "status": "Internal Server Error", "code": 500, @@ -6938,6 +8506,153 @@ ] } ] + }, + { + "name": "Data Analyze PII", + "item": [ + { + "name": "Data Analyze PII", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } + }, + "response": [ + { + "name": "Success - Data Analyze PII", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "description": "", + "type": "text" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-16T10:27:56+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"8fd2cfe2-7912-40d1-8710-9b7da9a35bd9\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"field\": \"primary_passenger.email\",\n \"type\": \"internet\",\n \"score\": 0.5,\n \"reason\": [\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m014\",\n \"region\": \"\",\n \"score\": 1\n },\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m004\",\n \"region\": \"\",\n \"score\": 1\n }\n ]\n },\n {\n \"field\": \"primary_passenger.mobile\",\n \"type\": \"phone\",\n \"score\": 0.5,\n \"reason\": [\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m018\",\n \"region\": \"USA\",\n \"score\": 1\n },\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m005\",\n \"region\": \"\",\n \"score\": 1\n }\n ]\n }\n ]\n}" + }, + { + "name": "Failure - Data Analyze PII", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } + }, + "status": "Internal Server Error", + "code": 500, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "description": "", + "type": "text" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-16T10:29:58+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75aa71f9-ef84-41b9-ae1b-64cc7afd3ffb\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"ERR_BAD_REQUEST\",\n \"message\": \"Request failed with status code 401\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] } ] } \ No newline at end of file From 31339e8b888db4fe9c8d7a4bf4344aea8fd85a54 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 16 Jan 2025 11:48:22 +0530 Subject: [PATCH 311/311] feat #OBS-I501 : Added swagger doc for pii api --- api-service/swagger-doc/openapi_v2.yml | 104 +++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index 8a65e8e0..1fcdf368 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -10309,4 +10309,108 @@ paths: error: code: FAILED_TO_FETCH_METRICS message: Error while fetching metrics + trace: '' + /v2/data/analyze/pii: + post: + tags: + - Data Analyze PII + summary: Data Analyze PII + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.analyze.pii + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + request: + id: 268cb552-7de9-41cc-a413-a8bd3b64734c + dataset_id: taxi-payment-data + data: + - tripID: f0f5186b-3195-486e-9b79-0cd9da895707 + VendorID: '1' + tpep_pickup_datetime: '2024-10-31 00:46:40' + tpep_dropoff_datetime: '2024-10-31 00:53:20' + passenger_count: '1' + trip_distance: '1.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '151' + DOLocationID: '239' + payment_type: '1' + primary_passenger.email: Herminia_Veum@hotmail.com + primary_passenger.mobile: 200-329-2445 x3611 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Authorization + in: header + schema: + type: string + example: Bearer {{token}} + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: api.data.analyze.pii + ver: v2 + ts: '2025-01-16T10:27:56+05:30' + params: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 8fd2cfe2-7912-40d1-8710-9b7da9a35bd9 + responseCode: OK + result: + - field: primary_passenger.email + type: internet + score: 0.5 + reason: + - code: en + resourceKey: pii.descriptions.m014 + region: '' + score: 1 + - code: en + resourceKey: pii.descriptions.m004 + region: '' + score: 1 + - field: primary_passenger.mobile + type: phone + score: 0.5 + reason: + - code: en + resourceKey: pii.descriptions.m018 + region: USA + score: 1 + - code: en + resourceKey: pii.descriptions.m005 + region: '' + score: 1 + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + example: + id: api.data.analyze.pii + ver: v2 + ts: '2025-01-16T10:29:58+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75aa71f9-ef84-41b9-ae1b-64cc7afd3ffb + responseCode: INTERNAL_SERVER_ERROR + error: + code: ERR_BAD_REQUEST + message: Request failed with status code 401 trace: '' \ No newline at end of file