diff --git a/api-service/.dockerignore b/api-service/.dockerignore index b512c09d..b83f1fd0 100644 --- a/api-service/.dockerignore +++ b/api-service/.dockerignore @@ -1 +1,3 @@ -node_modules \ No newline at end of file +node_modules +api-service/postman-collection +postman-collection \ No newline at end of file diff --git a/api-service/Dockerfile b/api-service/Dockerfile index 2d0f168d..76e7174f 100644 --- a/api-service/Dockerfile +++ b/api-service/Dockerfile @@ -1,7 +1,8 @@ -FROM --platform=linux/amd64 node:20-alpine +FROM --platform=linux/amd64 node:23.4-alpine RUN mkdir -p /opt/api-service COPY ./api-service ./opt/api-service WORKDIR /opt/api-service +RUN rm -rf postman-collection RUN npm install COPY . . EXPOSE 3000 diff --git a/api-service/package.json b/api-service/package.json index 782cda64..e80d65f7 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -22,6 +22,17 @@ "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", "@jsonhero/schema-infer": "^0.1.5", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.53.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.53.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-logs": "^0.53.0", + "@opentelemetry/sdk-metrics": "^1.26.0", + "@opentelemetry/sdk-node": "^0.53.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/sdk-trace-node": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", @@ -31,6 +42,7 @@ "busboy": "^1.6.0", "compression": "^1.7.4", "dateformat": "2.0.0", + "dayjs": "^1.11.13", "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", diff --git a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json index b48c505b..cd8d4689 100644 --- a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -1,11 +1,11 @@ { "info": { - "_postman_id": "d3ffc748-3d71-4395-9298-e065845f7bfb", + "_postman_id": "b94beefb-2a07-49ac-bd5b-07d66fe7ba85", "name": "Obsrv v2 apis", "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", - "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "26192103", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "406673", + "_collection_link": "https://galactic-flare-389256.postman.co/workspace/config-service-workspace~0ba551f8-be4b-4755-9d4d-bf4578113e5f/collection/406673-b94beefb-2a07-49ac-bd5b-07d66fe7ba85?action=share&source=collection_link&creator=406673" }, "item": [ { @@ -23,14 +23,25 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/datasets/create", + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + }, "description": "This API allows you to create new datasets used by the analytical data source." }, "response": [ @@ -53,7 +64,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -110,7 +132,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -167,7 +200,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -224,7 +268,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -281,7 +336,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Bad Request", "code": 400, @@ -338,7 +404,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -395,7 +472,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -454,7 +542,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url", + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + }, "description": "This API generates presigned URLs to upload or download files from cloud" }, "response": [ @@ -477,7 +576,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "OK", "code": 200, @@ -535,7 +645,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "OK", "code": 200, @@ -593,7 +714,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "Bad Request", "code": 400, @@ -651,7 +783,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "Bad Request", "code": 400, @@ -710,7 +853,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update", + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + }, "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors" }, "response": [ @@ -733,7 +887,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "OK", "code": 200, @@ -790,7 +955,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "OK", "code": 200, @@ -847,7 +1023,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Conflict", "code": 409, @@ -904,7 +1091,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Not Found", "code": 404, @@ -961,7 +1159,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Bad Request", "code": 400, @@ -1018,7 +1227,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Bad Request", "code": 400, @@ -1100,7 +1320,19 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3000/v2/datasets/read/master-test" + "url": { + "raw": "localhost:3000/v2/datasets/read/master-test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "read", + "master-test" + ] + } }, "status": "OK", "code": 200, @@ -1215,7 +1447,7 @@ } ], "url": { - "raw": "localhost:3000/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "raw": "localhost:3000/v2/datasets/read/master-test?fields=name,type,id", "host": [ "localhost" ], @@ -1227,10 +1459,6 @@ "master-test" ], "query": [ - { - "key": "status", - "value": "Draft" - }, { "key": "fields", "value": "name,type,id" @@ -1486,7 +1714,19 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3000/v2/datasets/read/new_telemetry_record.1" + "url": { + "raw": "localhost:3000/v2/datasets/read/new_telemetry_record.1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "read", + "new_telemetry_record.1" + ] + } }, "status": "Not Found", "code": 404, @@ -1540,7 +1780,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list", + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + }, "description": "This API allows you to list all datasets. User can apply filters on dataset status and type." }, "response": [ @@ -1558,7 +1809,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1610,7 +1872,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1649,7 +1922,7 @@ "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" }, { - "name": "Success: Filter basen on status as string", + "name": "Success: Filter based on status as string", "originalRequest": { "method": "POST", "header": [], @@ -1662,11 +1935,22 @@ } } }, - "url": "localhost:3000/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", @@ -1714,7 +1998,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1766,7 +2061,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "Bad Request", "code": 400, @@ -1829,7 +2135,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema", + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + }, "description": "This api is used to generate data schema for the given dataset event." }, "response": [ @@ -1856,7 +2173,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "OK", "code": 200, @@ -1917,7 +2245,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "Bad Request", "code": 400, @@ -1978,7 +2317,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "Bad Request", "code": 400, @@ -2032,7 +2382,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition", + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + }, "description": "This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset." }, "response": [ @@ -2050,14 +2411,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset deleted successfully\",\n \"dataset_id\": \"master.1\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Delete successful\",\n \"dataset_id\": \"trip-data\"\n }\n}" }, { "name": "Delete failure: Dataset not found to delete", @@ -2073,14 +2445,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to delete\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: master\",\n \"trace\": \"\"\n }\n}" }, { "name": "Live success: Dataset published successfully", @@ -2096,14 +2479,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset published successfully\",\n \"dataset_id\": \"telemetry\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Live successful\",\n \"dataset_id\": \"telemetry_record\"\n }\n}" }, { "name": "Live failure: Dataset not found to publish", @@ -2119,14 +2513,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to publish\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: telemetry-data\",\n \"trace\": \"\"\n }\n}" }, { "name": "Live failure: Dataset in draft state", @@ -2142,14 +2547,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_NOT_READY_FOR_PUBLISH\",\n \"message\": \"Failed to publish dataset as it is in draft state\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_LIVE_FAILURE\",\n \"message\": \"Transition failed for dataset: sb-telemetry2 status:Draft with status transition to Live\",\n \"trace\": \"\"\n }\n}" }, { "name": "ReadyToPublish success: Dataset is ready to publish", @@ -2165,7 +2581,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, @@ -2188,7 +2615,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2211,14 +2649,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Failed to mark dataset Ready to publish as it not in draft state\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Transition failed for dataset: telemetry-events status:Retired with status transition to ReadyToPublish\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire success: Dataset retired successfully", @@ -2234,14 +2683,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset retired successfully\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Retire successful\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" }, { "name": "Retire Failure: Dataset not found to retire", @@ -2257,14 +2717,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to retire\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: sb-telemetry2\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire Failure: Dataset is already retired", @@ -2280,14 +2751,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_ALREADY_RETIRED\",\n \"message\": \"Dataset is already retired\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_RETIRE_FAILURE\",\n \"message\": \"Transition failed for dataset: master-telemetrry status:Retired with status transition to Retire\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire failure: Cannot retire master dataset as it is used by other datasets", @@ -2303,14 +2785,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is used by other datasets\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset\",\n \"trace\": \"\"\n }\n}" }, { "name": "Failure: Invalid request payload provided", @@ -2326,7 +2819,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2334,29 +2838,6 @@ "header": [], "cookie": [], "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:03:56+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"ba4c86bd-b438-4582-b178-2410a5c5dd15\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_STATUS_INVALID_INPUT\",\n \"message\": \"#properties/request/properties/status/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to the db failed", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Delete\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/datasets/status-transition" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:24:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"92928434-719f-47d4-9946-1e40ecd53253\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"DATASET_STATUS_FAILURE\",\n \"message\": \"Failed to perform status transition on datasets\",\n \"trace\": \"\"\n }\n}" } ] }, @@ -2372,7 +2853,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" @@ -2380,7 +2861,7 @@ } }, "url": { - "raw": "localhost:3000/v2/datasets/import?override=true", + "raw": "localhost:3000/v2/datasets/import?overwrite=false", "host": [ "localhost" ], @@ -2392,105 +2873,53 @@ ], "query": [ { - "key": "override", - "value": "true" + "key": "overwrite", + "value": "false" } ] } }, - "response": [] - }, - { - "name": "Dataset export", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "datasets", - "export", - "v1-copy" - ], - "query": [ - { - "key": "status", - "value": "Live" - } - ] - } - }, - "response": [] - }, - { - "name": "Dataset copy", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/datasets/copy" - }, - "response": [] - } - ], - "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." - }, - { - "name": "Connector api's", - "item": [ - { - "name": "Connector list", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, "response": [ { - "name": "Failure: Invalid request body, filter option array should not be empty", + "name": "Success: Fully imported dataset", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2503,15 +2932,15 @@ }, { "key": "Content-Length", - "value": "399" + "value": "346" }, { "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + "value": "W/\"15a-SX5o6plRdfy+akEYhbMwq9zl+oU\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" + "value": "Tue, 19 Nov 2024 07:34:26 GMT" }, { "key": "Connection", @@ -2523,23 +2952,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:04:26+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"fd87393e-4b1b-4cf0-8e9a-a0529d691cc3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is imported successfully\",\n \"data\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"version_key\": \"1732001666671\"\n }\n }\n}" }, { - "name": "Success: Filtered based on status", + "name": "Success: Partially imported dataset", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } }, "status": "OK", "code": 200, @@ -2555,15 +3006,15 @@ }, { "key": "Content-Length", - "value": "250" + "value": "520" }, { "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + "value": "W/\"208-vju3WNZ7seMZsQ5AS29Wk0j+yOA\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" + "value": "Tue, 19 Nov 2024 07:29:41 GMT" }, { "key": "Connection", @@ -2575,23 +3026,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:59:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"379ced0b-eb06-445f-a7e3-8737cb16c351\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is partially imported\",\n \"data\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"version_key\": \"1732001381825\"\n },\n \"ignoredFields\": {\n \"denorm_fields\": [\n {\n \"config\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-dataset\"\n },\n \"details\": \"Master dataset does not exist\"\n }\n ]\n }\n }\n}" }, { - "name": "Success: Filtered based on category", + "name": "Success: Imported new dataset [overwrite=false]", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft\",\n \"dataset_id\": \"sb-telemetryRPF_Draft\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, "status": "OK", "code": 200, @@ -2607,15 +3080,15 @@ }, { "key": "Content-Length", - "value": "2571" + "value": "341" }, { "key": "ETag", - "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + "value": "W/\"155-UdyHMmAFuHt3xNP/3pFL5e1esfA\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:25:03 GMT" + "value": "Tue, 19 Nov 2024 07:35:32 GMT" }, { "key": "Connection", @@ -2627,26 +3100,48 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:05:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"1a50e1b1-14b0-4178-aba4-9e09a63d8d1d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is imported successfully\",\n \"data\": {\n \"id\": \"sb-telemetry_draft\",\n \"version_key\": \"1732001732045\"\n }\n }\n}" }, { - "name": "Success: Connectors list with all filter options", + "name": "Failure: Import Dataset with id already exists", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, - "status": "OK", - "code": 200, + "status": "Conflict", + "code": 409, "_postman_previewlanguage": "json", "header": [ { @@ -2659,15 +3154,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "351" }, { "key": "ETag", - "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + "value": "W/\"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:26:32 GMT" + "value": "Tue, 19 Nov 2024 07:31:11 GMT" }, { "key": "Connection", @@ -2679,79 +3174,48 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:01:11+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"1e5b34cd-4473-4439-8cc5-5ff917f47e48\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset with dataset_id: sb-telemetryRPF_draft_data already exists.\"\n }\n}" }, { - "name": "Success: Connectors list without filters", + "name": "Failure: Invalid configs to import", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4380" - }, - { - "key": "ETag", - "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" - }, - { - "key": "Date", - "value": "Wed, 31 Jul 2024 13:27:37 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - } - ] - }, - { - "name": "Connector Read", - "request": { - "method": "GET", - "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" - }, - "response": [ - { - "name": "Success: Read live Connector", - "originalRequest": { - "method": "GET", - "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -2764,15 +3228,15 @@ }, { "key": "Content-Length", - "value": "3304" + "value": "364" }, { "key": "ETag", - "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + "value": "W/\"16c-Dxem7prJM89DOqth+bdZbh0UI1g\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 12:47:54 GMT" + "value": "Tue, 19 Nov 2024 07:38:05 GMT" }, { "key": "Connection", @@ -2784,30 +3248,143 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:08:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"244318a5-d933-41a5-add6-14f299281069\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_IMPORT_INVALID_CONFIGS\",\n \"message\": \"#properties/request/required must have required property 'id'\"\n }\n}" }, { - "name": "Success: Read Draft Connector", + "name": "Dataset import", "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", - "protocol": "http", - "host": [ - "localhost" - ], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], "port": "3000", "path": [ "v2", - "connectors", - "read", - "mssql-connector-2.0.0" + "datasets", + "import" ], "query": [ { - "key": "mode", - "value": "edit" + "key": "overwrite", + "value": "true" + } + ] + } + }, + "_postman_previewlanguage": "Text", + "header": [], + "cookie": [], + "body": "" + }, + { + "name": "Dataset import", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } + }, + "_postman_previewlanguage": "Text", + "header": [], + "cookie": [], + "body": "" + } + ] + }, + { + "name": "Dataset export", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [ + { + "name": "Success: Live Dataset export success", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/content-data-v2?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "content-data-v2" + ], + "query": [ + { + "key": "status", + "value": "Live" } ] } @@ -2826,15 +3403,15 @@ }, { "key": "Content-Length", - "value": "744" + "value": "5155" }, { "key": "ETag", - "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + "value": "W/\"1423-Tx46QCrIX8So3vSGVkd865IYeEY\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 07:17:12 GMT" + "value": "Tue, 19 Nov 2024 07:16:01 GMT" }, { "key": "Connection", @@ -2846,17 +3423,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:46:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ae57c620-2858-40f9-9d8e-33f8f259449d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"content-data-v2\",\n \"dataset_id\": \"content-data-v2\",\n \"type\": \"master\",\n \"name\": \"content-data-v2\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"key\": \"tripID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"VendorID\": {\n \"key\": \"VendorID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tpep_pickup_datetime\": {\n \"key\": \"tpep_pickup_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"tpep_dropoff_datetime\": {\n \"key\": \"tpep_dropoff_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"passenger_count\": {\n \"key\": \"passenger_count\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"trip_distance\": {\n \"key\": \"trip_distance\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"RatecodeID\": {\n \"key\": \"RatecodeID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"store_and_fwd_flag\": {\n \"key\": \"store_and_fwd_flag\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"PULocationID\": {\n \"key\": \"PULocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"DOLocationID\": {\n \"key\": \"DOLocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"payment_type\": {\n \"key\": \"payment_type\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"primary_passenger\": {\n \"key\": \"primary_passenger\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"email\": {\n \"key\": \"email\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mobile\": {\n \"key\": \"mobile\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n }\n },\n \"fare_details\": {\n \"key\": \"fare_details\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"fare_amount\": {\n \"key\": \"fare_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"extra\": {\n \"key\": \"extra\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mta_tax\": {\n \"key\": \"mta_tax\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tip_amount\": {\n \"key\": \"tip_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tolls_amount\": {\n \"key\": \"tolls_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"improvement_surcharge\": {\n \"key\": \"improvement_surcharge\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"total_amount\": {\n \"key\": \"total_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"congestion_surcharge\": {\n \"key\": \"congestion_surcharge\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n }\n }\n },\n \"additionalProperties\": true\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"tripID\",\n \"dedup_period\": 604800\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"content-data-v2\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"container/api-service/user_uploads/chunk-2_addedf.json\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 7\n }\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"data_version\": 1,\n \"api_version\": \"v2\",\n \"version\": 1,\n \"sample_data\": {\n \"mergedEvent\": {\n \"tripID\": \"ccbb970c-4d10-4247-be0c-a480996f3429\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-26 00:36:10\",\n \"tpep_dropoff_datetime\": \"2023-09-15 00:45:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"263\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jacey_Hintz@yahoo.com\",\n \"mobile\": \"247-492-3370\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.25\",\n \"congestion_surcharge\": \"\"\n }\n }\n },\n \"entry_topic\": \"local.ingest\",\n \"created_date\": \"2024-10-14T06:23:44.588Z\",\n \"updated_date\": \"2024-10-14T06:23:44.588Z\"\n }\n}" }, { - "name": "Failure: Connector not found", + "name": "Success: ReadyToPublish Dataset exported successfully", "originalRequest": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + "url": { + "raw": "localhost:3000/v2/datasets/export/mydataset?status=Draft", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "mydataset" + ], + "query": [ + { + "key": "status", + "value": "Draft" + } + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2869,15 +3464,15 @@ }, { "key": "Content-Length", - "value": "276" + "value": "5635" }, { "key": "ETag", - "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + "value": "W/\"1603-tBOO40ZL+TVYU+yC9CWRvXGb7vs\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 09:32:48 GMT" + "value": "Tue, 19 Nov 2024 07:18:11 GMT" }, { "key": "Connection", @@ -2889,58 +3484,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data Ingest", - "item": [ - { - "name": "Data ingest", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:48:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"222847a0-32bf-4c63-be68-76f0e51258af\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mydataset\",\n \"dataset_id\": \"mydataset\",\n \"name\": \"mydataset\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"key\": \"tripID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"VendorID\": {\n \"key\": \"VendorID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tpep_pickup_datetime\": {\n \"key\": \"tpep_pickup_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"tpep_dropoff_datetime\": {\n \"key\": \"tpep_dropoff_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"passenger_count\": {\n \"key\": \"passenger_count\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"trip_distance\": {\n \"key\": \"trip_distance\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"RatecodeID\": {\n \"key\": \"RatecodeID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"store_and_fwd_flag\": {\n \"key\": \"store_and_fwd_flag\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"PULocationID\": {\n \"key\": \"PULocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"DOLocationID\": {\n \"key\": \"DOLocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"payment_type\": {\n \"key\": \"payment_type\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"primary_passenger\": {\n \"key\": \"primary_passenger\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"email\": {\n \"key\": \"email\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mobile\": {\n \"key\": \"mobile\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n },\n \"additionalProperties\": false\n },\n \"fare_details\": {\n \"key\": \"fare_details\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"fare_amount\": {\n \"key\": \"fare_amount\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"integer\"\n },\n \"1\": {\n \"type\": \"number\"\n }\n }\n },\n \"extra\": {\n \"key\": \"extra\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"mta_tax\": {\n \"key\": \"mta_tax\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tip_amount\": {\n \"key\": \"tip_amount\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"tolls_amount\": {\n \"key\": \"tolls_amount\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"integer\"\n },\n \"1\": {\n \"type\": \"number\"\n }\n }\n },\n \"improvement_surcharge\": {\n \"key\": \"improvement_surcharge\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"total_amount\": {\n \"key\": \"total_amount\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"congestion_surcharge\": {\n \"key\": \"congestion_surcharge\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true\n }\n },\n \"additionalProperties\": false\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"mydataset\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"container/api-service/user_uploads/chunk-1 (4) (1)_5ec855.json\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1731420131072\",\n \"api_version\": \"v2\",\n \"sample_data\": {\n \"mergedEvent\": {\n \"tripID\": \"4c77e9d5-538d-4eb7-8db1-4c2c32860aa8\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-11-02 00:18:42\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willa67@gmail.com\",\n \"mobile\": \"1-720-981-6399 x77055\"\n },\n \"fare_details\": {\n \"fare_amount\": 7,\n \"extra\": 0.5,\n \"mta_tax\": 0.5,\n \"tip_amount\": 0,\n \"tolls_amount\": 0,\n \"improvement_surcharge\": 0.3,\n \"total_amount\": 8.3,\n \"congestion_surcharge\": 0\n }\n }\n },\n \"entry_topic\": \"local.ingest\",\n \"created_date\": \"2024-11-11T21:31:21.390Z\",\n \"updated_date\": \"2024-11-12T08:32:27.931Z\"\n }\n}" }, - "url": "localhost:3000/v2/data/in/sb-telemetry-imported" - }, - "response": [ { - "name": "Entry topic not found", + "name": "Failure: Dataset not found to export", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" } - } - }, - "url": "localhost:3000/v2/data/in/sample-test" + ] + } }, "status": "Not Found", "code": 404, @@ -2956,15 +3525,15 @@ }, { "key": "Content-Length", - "value": "316" + "value": "301" }, { "key": "ETag", - "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" + "value": "W/\"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ\"" }, { "key": "Date", - "value": "Thu, 18 Apr 2024 10:17:00 GMT" + "value": "Tue, 19 Nov 2024 07:15:10 GMT" }, { "key": "Connection", @@ -2976,29 +3545,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:45:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"fd6fdae2-a002-4c15-81e2-3cce539a2a5a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:v1-copy not found to export\"\n }\n}" }, { - "name": "Failure: schema validation", + "name": "Failure: Invalid dataset state to export", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", - "options": { - "raw": { - "language": "json" + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/new-dataset?status=Draft", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "new-dataset" + ], + "query": [ + { + "key": "status", + "value": "Draft" } - } - }, - "url": "localhost:3000/v2/data/in/test2" + ] + } }, "status": "Bad Request", "code": 400, @@ -3014,15 +3586,15 @@ }, { "key": "Content-Length", - "value": "300" + "value": "291" }, { "key": "ETag", - "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" + "value": "W/\"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:03:32 GMT" + "value": "Tue, 19 Nov 2024 07:20:43 GMT" }, { "key": "Connection", @@ -3034,29 +3606,74 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Success: Data ingested successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:50:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"2097228c-7872-4217-a06a-0ec08e3dc67d\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXPORT_FAILURE\",\n \"message\": \"Dataset with status:Draft cannot be exported\"\n }\n}" + } + ] + }, + { + "name": "Dataset copy", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } + }, + "response": [ + { + "name": "Success: Dataset cloned successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"content-data-v2\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"new-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, "status": "OK", "code": 200, @@ -3072,15 +3689,15 @@ }, { "key": "Content-Length", - "value": "261" + "value": "296" }, { "key": "ETag", - "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" + "value": "W/\"128-bi4wSfgkyMP13qFE8VPTmgDdgLA\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:05:48 GMT" + "value": "Tue, 19 Nov 2024 07:26:12 GMT" }, { "key": "Connection", @@ -3092,32 +3709,42 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:56:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"28072631-8e80-45eb-906f-d933a90646d0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"new-copy-live2\",\n \"message\": \"Dataset clone successful\"\n }\n}" }, { - "name": "Success: Data ingested successfully (batch)", + "name": "Failure: Dataset not found to copy", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", - "value": "application/json", - "type": "text" + "value": "application/json" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"content-data-v2\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"new-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test2" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3130,15 +3757,15 @@ }, { "key": "Content-Length", - "value": "261" + "value": "328" }, { "key": "ETag", - "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" + "value": "W/\"148-jUDshghYf/jOhmmVkvtHlfJT864\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:06:49 GMT" + "value": "Tue, 19 Nov 2024 07:22:41 GMT" }, { "key": "Connection", @@ -3150,32 +3777,75 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:52:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"f2fd51bb-fc18-4278-a229-47fa25398e69\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset dataset-telemetry does not exists\"\n }\n}" + } + ] + } + ], + "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } + }, + "response": [ { - "name": "Failure: Dataset not found", + "name": "Failure: Invalid request body, filter option array should not be empty", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/added-tags" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -3188,15 +3858,15 @@ }, { "key": "Content-Length", - "value": "317" + "value": "399" }, { "key": "ETag", - "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:07:29 GMT" + "value": "Tue, 30 Jul 2024 09:43:14 GMT" }, { "key": "Connection", @@ -3208,32 +3878,38 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" }, { - "name": "Failure: Entry topic not found", + "name": "Success: Filtered based on status", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/sample1" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3246,15 +3922,15 @@ }, { "key": "Content-Length", - "value": "316" + "value": "250" }, { "key": "ETag", - "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:09:26 GMT" + "value": "Tue, 30 Jul 2024 09:55:51 GMT" }, { "key": "Connection", @@ -3266,60 +3942,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data query", - "item": [ - { - "name": "Data query", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" }, - "url": "localhost:3000/v2/data/query/test" - }, - "response": [ { - "name": "Success: native query (interval as array type)", + "name": "Success: Filtered based on category", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3335,15 +3986,15 @@ }, { "key": "Content-Length", - "value": "313" + "value": "2571" }, { "key": "ETag", - "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:17:45 GMT" + "value": "Wed, 31 Jul 2024 13:25:03 GMT" }, { "key": "Connection", @@ -3355,30 +4006,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" }, { - "name": "Success: native query (queryType: scan)", + "name": "Success: Connectors list with all filter options", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3394,15 +4050,15 @@ }, { "key": "Content-Length", - "value": "313" + "value": "4380" }, { "key": "ETag", - "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:21:14 GMT" + "value": "Wed, 31 Jul 2024 13:26:32 GMT" }, { "key": "Connection", @@ -3414,30 +4070,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" }, { - "name": "Success: native query (normal intervals & queryType : timeseries)", + "name": "Success: Connectors list without filters", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3453,15 +4114,15 @@ }, { "key": "Content-Length", - "value": "289" + "value": "4380" }, { "key": "ETag", - "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:21:49 GMT" + "value": "Wed, 31 Jul 2024 13:27:37 GMT" }, { "key": "Connection", @@ -3473,33 +4134,53 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" - }, + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ] + } + }, + "response": [ { - "name": "Failure: Datasource not found in live table", + "name": "Success: Read live Connector", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/data/query/telemetry-eventssss" + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3512,15 +4193,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "3304" }, { "key": "ETag", - "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:13:12 GMT" + "value": "Wed, 31 Jul 2024 12:47:54 GMT" }, { "key": "Connection", @@ -3532,33 +4213,36 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" }, { - "name": "Failure: Invalid date range", + "name": "Success: Read Draft Connector", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "mssql-connector-2.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" } - } - }, - "url": "localhost:3000/v2/data/query/telemetry-events" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3571,15 +4255,15 @@ }, { "key": "Content-Length", - "value": "371" + "value": "744" }, { "key": "ETag", - "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:16:29 GMT" + "value": "Thu, 01 Aug 2024 07:17:12 GMT" }, { "key": "Connection", @@ -3591,30 +4275,27 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" }, { - "name": "Failure: Datasource not found in druid", + "name": "Failure: Connector not found", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/data/query/test" + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-conn", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-conn" + ] + } }, "status": "Not Found", "code": 404, @@ -3630,15 +4311,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "276" }, { "key": "ETag", - "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:20:27 GMT" + "value": "Thu, 01 Aug 2024 09:32:48 GMT" }, { "key": "Connection", @@ -3650,33 +4331,85 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Ingest", + "item": [ + { + "name": "Data ingest", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "localhost:3000/v2/data/in/sb-telemetry-imported", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sb-telemetry-imported" + ] + } + }, + "response": [ { - "name": "Success: sql query", + "name": "Entry topic not found", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/in/sample-test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sample-test" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3689,15 +4422,15 @@ }, { "key": "Content-Length", - "value": "582" + "value": "316" }, { "key": "ETag", - "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" + "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:12:12 GMT" + "value": "Thu, 18 Apr 2024 10:17:00 GMT" }, { "key": "Connection", @@ -3709,33 +4442,44 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" }, { - "name": "SQL Failure: dataset not found in druid", + "name": "Failure: schema validation", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/in/test2", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test2" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -3748,15 +4492,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "300" }, { "key": "ETag", - "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" + "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:22:26 GMT" + "value": "Thu, 30 May 2024 18:03:32 GMT" }, { "key": "Connection", @@ -3768,33 +4512,44 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" }, { - "name": "SQL Failure: Datasource not found in live table", + "name": "Success: Data ingested successfully", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/undefined" + "url": { + "raw": "localhost:3000/v2/data/in/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3807,15 +4562,15 @@ }, { "key": "Content-Length", - "value": "346" + "value": "261" }, { "key": "ETag", - "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" + "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:23:02 GMT" + "value": "Thu, 30 May 2024 18:05:48 GMT" }, { "key": "Connection", @@ -3827,69 +4582,41 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data exhaust", - "item": [ - { - "name": "Data exhaust", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "data", - "exhaust", - "beckn-test-data" - ], - "query": [ - { - "key": "from", - "value": "2024-02-01" - }, - { - "key": "to", - "value": "2024-02-22" - }, - { - "key": "type", - "value": "raw" - } - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Query Templates", - "item": [ - { - "name": "Read template", - "request": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql_template_11" - }, - "response": [ + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + }, { - "name": "Success: JSON template", + "name": "Success: Data ingested successfully (batch)", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/jsontemplate111" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/in/test2", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test2" + ] + } }, "status": "OK", "code": 200, @@ -3905,15 +4632,15 @@ }, { "key": "Content-Length", - "value": "537" + "value": "261" }, { "key": "ETag", - "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" + "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:24:14 GMT" + "value": "Thu, 30 May 2024 18:06:49 GMT" }, { "key": "Connection", @@ -3925,17 +4652,44 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" }, { - "name": "Success: SQL template", + "name": "Failure: Dataset not found", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql1" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/in/added-tags", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "added-tags" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3948,15 +4702,15 @@ }, { "key": "Content-Length", - "value": "528" + "value": "317" }, { "key": "ETag", - "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" + "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:24:38 GMT" + "value": "Thu, 30 May 2024 18:07:29 GMT" }, { "key": "Connection", @@ -3968,14 +4722,41 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure : Template not exists", + "name": "Failure: Entry topic not found", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql100" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/in/sample1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sample1" + ] + } }, "status": "Not Found", "code": 404, @@ -3991,15 +4772,15 @@ }, { "key": "Content-Length", - "value": "294" + "value": "316" }, { "key": "ETag", - "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:25:16 GMT" + "value": "Thu, 30 May 2024 18:09:26 GMT" }, { "key": "Connection", @@ -4011,24 +4792,84 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" } ] - }, + } + ] + }, + { + "name": "Data query", + "item": [ { - "name": "Delete template", + "name": "Data query", "request": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "response": [ { - "name": "Success: Deleted successfully", + "name": "Success: native query (interval as array type)", "originalRequest": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4044,15 +4885,15 @@ }, { "key": "Content-Length", - "value": "241" + "value": "313" }, { "key": "ETag", - "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" + "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:28:27 GMT" + "value": "Thu, 30 May 2024 18:17:45 GMT" }, { "key": "Connection", @@ -4064,17 +4905,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" }, { - "name": "Failure: Template does not exists", + "name": "Success: native query (queryType: scan)", "originalRequest": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/json_template" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4087,15 +4956,15 @@ }, { "key": "Content-Length", - "value": "303" + "value": "313" }, { "key": "ETag", - "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" + "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:28:41 GMT" + "value": "Thu, 30 May 2024 18:21:14 GMT" }, { "key": "Connection", @@ -4107,54 +4976,42 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Create query template", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" }, - "url": "localhost:3000/v2/template/create/json_template_111" - }, - "response": [ { - "name": "Success : Create query template success (json)", + "name": "Success: native query (normal intervals & queryType : timeseries)", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4170,15 +5027,15 @@ }, { "key": "Content-Length", - "value": "359" + "value": "289" }, { "key": "ETag", - "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" + "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 16:59:01 GMT" + "value": "Thu, 30 May 2024 18:21:49 GMT" }, { "key": "Connection", @@ -4190,32 +5047,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" }, { - "name": "Success: Create query template success (SQL)", + "name": "Failure: Datasource not found in live table", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/telemetry-eventssss", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "telemetry-eventssss" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4228,15 +5098,15 @@ }, { "key": "Content-Length", - "value": "339" + "value": "356" }, { "key": "ETag", - "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" + "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 16:59:58 GMT" + "value": "Thu, 30 May 2024 18:13:12 GMT" }, { "key": "Connection", @@ -4248,32 +5118,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Template already exists", + "name": "Failure: Invalid date range", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "telemetry-events" + ] + } }, - "status": "Conflict", - "code": 409, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -4286,15 +5169,15 @@ }, { "key": "Content-Length", - "value": "350" + "value": "371" }, { "key": "ETag", - "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:03:28 GMT" + "value": "Thu, 30 May 2024 18:16:29 GMT" }, { "key": "Connection", @@ -4306,32 +5189,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Missing required variables", + "name": "Failure: Datasource not found in druid", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4344,15 +5240,15 @@ }, { "key": "Content-Length", - "value": "407" + "value": "356" }, { "key": "ETag", - "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:05:10 GMT" + "value": "Thu, 30 May 2024 18:20:27 GMT" }, { "key": "Connection", @@ -4364,37 +5260,50 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Schema validation failure", + "name": "Success: sql query", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" }, { "key": "Content-Type", @@ -4402,15 +5311,15 @@ }, { "key": "Content-Length", - "value": "416" + "value": "582" }, { "key": "ETag", - "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:13:57 GMT" + "value": "Thu, 30 May 2024 18:12:12 GMT" }, { "key": "Connection", @@ -4422,45 +5331,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "List templates", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" }, - "url": "localhost:3000/v2/template/list" - }, - "response": [ { - "name": "empty request body", + "name": "SQL Failure: dataset not found in druid", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4473,15 +5382,15 @@ }, { "key": "Content-Length", - "value": "6864" + "value": "356" }, { "key": "ETag", - "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 13:46:03 GMT" + "value": "Thu, 30 May 2024 18:22:26 GMT" }, { "key": "Connection", @@ -4493,26 +5402,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success : order by", + "name": "SQL Failure: Datasource not found in live table", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/data/query/undefined", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "undefined" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4525,15 +5453,15 @@ }, { "key": "Content-Length", - "value": "7700" + "value": "346" }, { "key": "ETag", - "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:06:09 GMT" + "value": "Thu, 30 May 2024 18:23:02 GMT" }, { "key": "Connection", @@ -4545,23 +5473,93 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" - }, + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data exhaust", + "item": [ + { + "name": "Data exhaust", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "exhaust", + "beckn-test-data" + ], + "query": [ + { + "key": "from", + "value": "2024-02-01" + }, + { + "key": "to", + "value": "2024-02-22" + }, + { + "key": "type", + "value": "raw" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Query Templates", + "item": [ + { + "name": "Read template", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/template/read/sql_template_11", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql_template_11" + ] + } + }, + "response": [ { - "name": "Success : Filters", + "name": "Success: JSON template", "originalRequest": { - "method": "POST", + "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/read/jsontemplate111", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "jsontemplate111" + ] + } }, "status": "OK", "code": 200, @@ -4577,15 +5575,15 @@ }, { "key": "Content-Length", - "value": "2708" + "value": "537" }, { "key": "ETag", - "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:08:55 GMT" + "value": "Thu, 30 May 2024 18:24:14 GMT" }, { "key": "Connection", @@ -4597,23 +5595,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" }, { - "name": "Success : limit and offset", + "name": "Success: SQL template", "originalRequest": { - "method": "POST", + "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/read/sql1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql1" + ] + } }, "status": "OK", "code": 200, @@ -4629,15 +5630,15 @@ }, { "key": "Content-Length", - "value": "1921" + "value": "528" }, { "key": "ETag", - "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:11:36 GMT" + "value": "Thu, 30 May 2024 18:24:38 GMT" }, { "key": "Connection", @@ -4649,42 +5650,103 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" - } - ] - }, - { - "name": "update template", - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" }, - "url": "localhost:3000/v2/template/update/sql11template1" + { + "name": "Failure : Template not exists", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/template/read/sql100", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql100" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "294" + }, + { + "key": "ETag", + "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:25:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Delete template", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "localhost:3000/v2/template/delete/yashash-k", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "yashash-k" + ] + } }, "response": [ { - "name": "Success: update successful", + "name": "Success: Deleted successfully", "originalRequest": { - "method": "PATCH", + "method": "DELETE", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/delete/yashash-k", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "yashash-k" + ] + } }, "status": "OK", "code": 200, @@ -4700,15 +5762,15 @@ }, { "key": "Content-Length", - "value": "314" + "value": "241" }, { "key": "ETag", - "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:51:47 GMT" + "value": "Tue, 30 Apr 2024 06:28:27 GMT" }, { "key": "Connection", @@ -4720,26 +5782,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" }, { - "name": "Failure: required variables not exists to update", + "name": "Failure: Template does not exists", "originalRequest": { - "method": "PATCH", + "method": "DELETE", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/delete/json_template", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "json_template" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4752,15 +5817,15 @@ }, { "key": "Content-Length", - "value": "429" + "value": "303" }, { "key": "ETag", - "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:53:54 GMT" + "value": "Thu, 30 May 2024 18:28:41 GMT" }, { "key": "Connection", @@ -4772,26 +5837,79 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Create query template", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "response": [ { - "name": "Failure: Template name validation failure", + "name": "Success : Create query template success (json)", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4804,15 +5922,15 @@ }, { "key": "Content-Length", - "value": "405" + "value": "359" }, { "key": "ETag", - "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:56:59 GMT" + "value": "Mon, 29 Apr 2024 16:59:01 GMT" }, { "key": "Connection", @@ -4824,26 +5942,43 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" }, { - "name": "Failure: query_type should when updating query", + "name": "Success: Create query template success (SQL)", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4856,15 +5991,15 @@ }, { "key": "Content-Length", - "value": "412" + "value": "339" }, { "key": "ETag", - "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" + "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 07:02:57 GMT" + "value": "Mon, 29 Apr 2024 16:59:58 GMT" }, { "key": "Connection", @@ -4876,26 +6011,2212 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" }, { - "name": "Failure: query should present when updating query_type", + "name": "Failure: Template already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "350" + }, + { + "key": "ETag", + "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:03:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Missing required variables", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "407" + }, + { + "key": "ETag", + "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:05:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Schema validation failure", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "416" + }, + { + "key": "ETag", + "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:13:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "List templates", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "response": [ + { + "name": "empty request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "6864" + }, + { + "key": "ETag", + "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 13:46:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + }, + { + "name": "Success : order by", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "7700" + }, + { + "key": "ETag", + "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:06:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : Filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2708" + }, + { + "key": "ETag", + "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : limit and offset", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1921" + }, + { + "key": "ETag", + "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:11:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" + } + ] + }, + { + "name": "update template", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "response": [ + { + "name": "Success: update successful", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "314" + }, + { + "key": "ETag", + "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:51:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + }, + { + "name": "Failure: required variables not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "429" + }, + { + "key": "ETag", + "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:53:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Template name validation failure", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "405" + }, + { + "key": "ETag", + "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:56:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query_type should when updating query", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "412" + }, + { + "key": "ETag", + "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 07:02:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query should present when updating query_type", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "412" + }, + { + "key": "ETag", + "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 07:04:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "query template", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "response": [ + { + "name": "Failure: invalid date range (native template)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "telemetry-events" + ], + "query": [ + { + "key": "limit", + "value": "1", + "disabled": true + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "336" + }, + { + "key": "ETag", + "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + }, + { + "key": "Date", + "value": "Thu, 02 May 2024 07:29:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "357" + }, + { + "key": "ETag", + "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:51:46 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "343" + }, + { + "key": "ETag", + "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:53:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: invalid date range", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:58:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: JSON template with request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/jsontemplate1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "jsontemplate1" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "301" + }, + { + "key": "ETag", + "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" + }, + { + "key": "Date", + "value": "Tue, 14 May 2024 06:22:24 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + }, + { + "name": "Success: SQL template query with request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "594" + }, + { + "key": "ETag", + "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" + }, + { + "key": "Date", + "value": "Tue, 14 May 2024 06:03:06 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + } + ] + } + ] + }, + { + "name": "Schema validator", + "item": [ + { + "name": "Schema validator", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"test_dataset\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/data/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "data", + "v2", + "schema", + "validate" + ] + } + }, + "response": [ + { + "name": "Succes: Schema validation successful", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"sb-telemetry\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "schema", + "validate" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "266" + }, + { + "key": "ETag", + "value": "W/\"10a-hzmB3lhC1mzaRhnGxclXsZ1lfHY\"" + }, + { + "key": "Date", + "value": "Thu, 20 Jun 2024 11:19:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-20T16:49:21+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"af336811-cfd8-4436-9682-05e0f25e2575\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Success\",\n \"isValid\": true\n }\n}" + }, + { + "name": "Failure: Dataset not exists", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"test_dataset\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "schema", + "validate" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "324" + }, + { + "key": "ETag", + "value": "W/\"144-Eg0khgCz78o32V2ogAkVSj5KgLw\"" + }, + { + "key": "Date", + "value": "Tue, 18 Jun 2024 11:39:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T17:09:38+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5ad8657b-b556-4563-9d39-e1170f1062e8\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset test_dataset does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Alert Notification-channels", + "item": [ + { + "name": "search channels", + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "search" + ] + } + }, + "response": [] + }, + { + "name": "Add channel (Slack)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"manager\": \"grafana\",\n \"name\": \"functional-metrics-slack\",\n \"type\": \"slack\",\n \"config\": {\n \"channel\": \"grafana-alerts\",\n \"webhookUrl\": \"https://hooks.slack.com/services/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\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "create" + ] + } + }, + "response": [] + }, + { + "name": "publish channel", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/publish/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "publish", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } + }, + "response": [] + }, + { + "name": "test channel", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"message\": \"Testing Email integration. If you can read this, it's working!\",\n \"payload\": {\n \"error\": {\n \"selectChannel\": true,\n \"configureChannel\": true\n },\n \"manager\": \"grafana\",\n \"name\": \"udhw\",\n \"type\": \"email\",\n \"config\": {\n \"recipientAddresses\": \"jerald@sanketika.in\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/test", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "test" + ] + } + }, + "response": [] + }, + { + "name": "update channel", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"updated name\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/update/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "update", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } + }, + "response": [] + }, + { + "name": "delete channel", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/delete/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "delete", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } + }, + "response": [] + }, + { + "name": "Add channel (Email)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"manager\": \"grafana\",\n \"name\": \"functional-metrics-email\",\n \"type\": \"email\",\n \"config\": {\n \"recipientAddresses\": \"yravinderkumar33@gmail.com;ravinder@sanketika.in\",\n \"subject\": \"Obsrv Prod Alert\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "create" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Alert silence", + "item": [ + { + "name": "Add Silence", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"startDate\": \"2024-01-11T07:56:23Z\",\n \"endDate\": \"2024-01-11T08:56:23Z\",\n \"alertId\": \"c7464d32-1d8d-4eaf-9b23-1313a3ff8149\",\n \"manager\": \"grafana\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "create" + ] + } + }, + "response": [] + }, + { + "name": "Get Silence", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/get/0ec52a4c-4529-4f70-b6f1-0de4b1e81c13", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "get", + "0ec52a4c-4529-4f70-b6f1-0de4b1e81c13" + ] + } + }, + "response": [] + }, + { + "name": "Delete Silence", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/delete/39c7eec1-db78-4f5d-8e77-38385cc3b7dc", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "delete", + "39c7eec1-db78-4f5d-8e77-38385cc3b7dc" + ] + } + }, + "response": [] + }, + { + "name": "Fetch All Silences", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "search" + ] + } + }, + "response": [] + }, + { + "name": "Edit Request", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"startDate\": \"2023-08-09T10:50:59Z\",\n \"endDate\": \"2023-08-10T10:30:59Z\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/update/da46091b-65db-4df9-85d1-c2341168bcb8", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "update", + "da46091b-65db-4df9-85d1-c2341168bcb8" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Alerts Wrapper", + "item": [ + { + "name": "delete rule ", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/delete/0c3e08d4-8e4f-4810-b591-f5a294ee378e", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "delete", + "0c3e08d4-8e4f-4810-b591-f5a294ee378e" + ] + }, + "description": "This URLwill provided access to user to delete any custom rule." + }, + "response": [] + }, + { + "name": "edit rule ", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"description\":\"This alert rule is designed to promptly notify you when one or more servers in your infrastructure become unresponsive or inaccessible.\",\n \"labels\":{\n \"severity\":\"warning\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/update/a6c46c48-79c1-4e34-9552-fe8f13d3c73e", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "update", + "a6c46c48-79c1-4e34-9552-fe8f13d3c73e" + ] + }, + "description": "This URL will provide access to users to edit any properties in the Alert rule and save the changes." + }, + "response": [] + }, + { + "name": "publish rule ", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "" + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/publish/c6c93917-9e5e-4aa6-9986-1c53c45ba7fe", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "publish", + "c6c93917-9e5e-4aa6-9986-1c53c45ba7fe" + ] + } + }, + "response": [] + }, + { + "name": "Add Rule ", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*" + }, + { + "key": "Accept-Language", + "value": "en-GB,en" + }, + { + "key": "Cache-Control", + "value": "no-store" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; grafana_session=4eb514f6fef4ad6884e47e50254af650; grafana_session_expiry=1687239735" + }, + { + "key": "Origin", + "value": "http://localhost:3001" + }, + { + "key": "Pragma", + "value": "no-store" + }, + { + "key": "Referer", + "value": "http://localhost:3001/alertRules/add" + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty" + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors" + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin" + }, + { + "key": "Sec-GPC", + "value": "1" + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"Total Api Calls1\",\n \"manager\":\"grafana\",\n \"description\": \"This alert is set up to notify you when the CPU usage on a host reaches a Low level.\",\n \"expression\": \"(node_total_api_calls) > 20\",\n \"category\": \"Infra\",\n \"severity\":\"warning\",\n \"frequency\": \"1m\",\n \"interval\": \"1m\",\n \"labels\": {\n \"component\": \"api\",\n \"notificationChannel\": \"slack\"\n },\n \"annotations\":{\n \"summary\":\"Host Low CPU usage\"\n },\n \"metadata\": {\n \"query\": [\n {\n \"refId\": \"A\",\n \"datasourceUid\": \"$datasourceUid\",\n \"queryType\": \"\",\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n },\n \"model\": {\n \"refId\": \"A\",\n \"hide\": false,\n \"editorMode\": \"code\",\n \"expr\": \"node_total_api_calls\",\n \"legendFormat\": \"__auto\",\n \"range\": true\n }\n },\n {\n \"refId\": \"B\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"B\",\n \"hide\": false,\n \"type\": \"reduce\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"B\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"reducer\": \"last\",\n \"expression\": \"A\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n },\n {\n \"refId\": \"C\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"C\",\n \"hide\": false,\n \"type\": \"threshold\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [\n 20\n ],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"C\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"expression\": \"B\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n }\n ]\n }\n}" + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "create" + ] + } + }, + "response": [] + }, + { + "name": "search rule ", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "search" + ] + } + }, + "response": [] + }, + { + "name": "get specific rule ", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/get/dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "get", + "dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Alert Metric_alias", + "item": [ + { + "name": "add metric", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"alias\": \"Druid\",\n \"component\": \"MyTest\",\n \"metric\": \"sum(druid_supervisors{state='SUSPENDED'})\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "create" + ] + } + }, + "response": [] + }, + { + "name": "list metrics", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "search" + ] + } + }, + "response": [] + }, + { + "name": "update metric", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"alias\":\"Druid native\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/update/48aad878-d59c-4046-aeb5-fb87aefbcfed", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "update", + "48aad878-d59c-4046-aeb5-fb87aefbcfed" + ] + } + }, + "response": [] + }, + { + "name": "remove metric", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/delete/f168707f-5908-4e2e-9111-e73c41e2700d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "delete", + "f168707f-5908-4e2e-9111-e73c41e2700d" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Connector registry", + "item": [ + { + "name": "Connector registry upload", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream", + "type": "text", + "disabled": true + }, + { + "key": "Content-Type", + "value": "multipart/form-data", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file2", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/object_store_connector-0.1.0.tar", + "disabled": true + }, + { + "key": "file", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/zipfile.tar.gz", + "disabled": true + }, + { + "key": "file3", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/kafka-connector-1.0.0-distribution.tar.gz" + }, + { + "key": "file4", + "value": "hello", + "type": "text", + "disabled": true + } + ] + }, + "url": { + "raw": "localhost:3000/v2/connector/register", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connector", + "register" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Dataset metrics", + "item": [ + { + "name": "Dataset metrics", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } + }, + "response": [ + { + "name": "Failure: Dataset not found", "originalRequest": { - "method": "PATCH", + "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4908,15 +8229,15 @@ }, { "key": "Content-Length", - "value": "412" + "value": "330" }, { "key": "ETag", - "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" + "value": "W/\"14a-RKjp44K2gCKCdq3dClRy/aYxcd8\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 07:04:55 GMT" + "value": "Wed, 15 Jan 2025 09:45:30 GMT" }, { "key": "Connection", @@ -4928,41 +8249,16 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "query template", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:15:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"25107bba-6d88-4e9b-b951-61e29a72b97a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id test not found in live table\",\n \"trace\": \"\"\n }\n}" }, - "url": "localhost:3000/v2/template/query/sql_test_1" - }, - "response": [ { - "name": "Failure: invalid date range (native template)", + "name": "Failure: Request validation failed", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", "options": { "raw": { "language": "json" @@ -4970,23 +8266,15 @@ } }, "url": { - "raw": "localhost:3000/v2/template/query/telemetry-events", + "raw": "localhost:3007/v2/dataset/metrics", "host": [ "localhost" ], - "port": "3000", + "port": "3007", "path": [ "v2", - "template", - "query", - "telemetry-events" - ], - "query": [ - { - "key": "limit", - "value": "1", - "disabled": true - } + "dataset", + "metrics" ] } }, @@ -5004,15 +8292,15 @@ }, { "key": "Content-Length", - "value": "336" + "value": "362" }, { "key": "ETag", - "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + "value": "W/\"16a-pmYCarF3FSxd3J7zVBlhwST+rq8\"" }, { "key": "Date", - "value": "Thu, 02 May 2024 07:29:14 GMT" + "value": "Wed, 15 Jan 2025 09:53:22 GMT" }, { "key": "Connection", @@ -5024,26 +8312,37 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:23:22+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a6a26834-bb32-4e32-8dfd-473b0d21c8e8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Datasource not found in druid", + "name": "Success: Get all the data", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5056,15 +8355,15 @@ }, { "key": "Content-Length", - "value": "357" + "value": "1780" }, { "key": "ETag", - "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" + "value": "W/\"6f4-nUo39jrIJBlBgZ4cY4UL84W7UNU\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:51:46 GMT" + "value": "Wed, 15 Jan 2025 09:54:21 GMT" }, { "key": "Connection", @@ -5076,26 +8375,37 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:24:21+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ef86ff9c-4342-4cdc-a573-d469cbf436eb\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"category\": \"data_freshness\",\n \"status\": \"Healthy\",\n \"components\": [\n {\n \"type\": \"average_time_difference_in_min\",\n \"threshold\": 5,\n \"value\": 0,\n \"status\": \"Healthy\"\n },\n {\n \"type\": \"freshness_query_time_in_min\",\n \"threshold\": 10,\n \"value\": 0,\n \"status\": \"Healthy\"\n }\n ]\n },\n {\n \"category\": \"data_observability\",\n \"status\": \"Unhealthy\",\n \"components\": [\n {\n \"type\": \"data_observability_health\",\n \"status\": \"Unhealthy\"\n },\n {\n \"type\": \"failure_percentage\",\n \"value\": 35.39823008849557\n },\n {\n \"type\": \"threshold_percentage\",\n \"value\": 5\n },\n {\n \"type\": \"importance_score\",\n \"value\": 0\n }\n ]\n },\n {\n \"category\": \"data_volume\",\n \"components\": [\n {\n \"type\": \"events_per_hour\",\n \"value\": 0\n },\n {\n \"type\": \"events_per_day\",\n \"value\": 95\n },\n {\n \"type\": \"events_per_n_day\",\n \"value\": 765\n },\n {\n \"type\": \"volume_percentage_by_hour\",\n \"value\": 0\n },\n {\n \"type\": \"volume_percentage_by_day\",\n \"value\": -87.58169934640523\n },\n {\n \"type\": \"volume_percentage_by_week\",\n \"value\": 0\n },\n {\n \"type\": \"growth_rate_percentage\",\n \"value\": 0\n }\n ]\n },\n {\n \"category\": \"data_lineage\",\n \"components\": [\n {\n \"type\": \"transformation_success\",\n \"value\": 765\n },\n {\n \"type\": \"dedup_success\",\n \"value\": 384\n },\n {\n \"type\": \"denormalization_success\",\n \"value\": 572\n },\n {\n \"type\": \"total_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"transformation_failed\",\n \"value\": 1\n },\n {\n \"type\": \"dedup_failed\",\n \"value\": 574\n },\n {\n \"type\": \"denorm_failed\",\n \"value\": 193\n }\n ]\n },\n {\n \"category\": \"connectors\",\n \"components\": {\n \"category\": \"connectors\",\n \"components\": [\n {\n \"id\": \"failed\",\n \"type\": \"failed\",\n \"value\": 1608\n },\n {\n \"id\": \"api\",\n \"type\": \"success\",\n \"value\": 765\n }\n ]\n }\n },\n {\n \"category\": \"data_quality\",\n \"components\": [\n {\n \"type\": \"incidents_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"incidents_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_incidents\",\n \"value\": 2373\n }\n ]\n }\n ]\n}" }, { - "name": "Failure: Datasource not found in live table", + "name": "Success : Get particular category data", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n // \"data_freshness\",\n // \"data_observability\",\n // \"data_volume\",\n \"data_lineage\"\n // \"connectors\",\n // \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5108,15 +8418,15 @@ }, { "key": "Content-Length", - "value": "343" + "value": "594" }, { "key": "ETag", - "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" + "value": "W/\"252-mKqBPcpjk0LPbqNHyghckQWQnHs\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:53:47 GMT" + "value": "Wed, 15 Jan 2025 09:55:13 GMT" }, { "key": "Connection", @@ -5128,26 +8438,37 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:25:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"10915ec3-a265-4502-bc9b-673562f94512\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"category\": \"data_lineage\",\n \"components\": [\n {\n \"type\": \"transformation_success\",\n \"value\": 765\n },\n {\n \"type\": \"dedup_success\",\n \"value\": 384\n },\n {\n \"type\": \"denormalization_success\",\n \"value\": 572\n },\n {\n \"type\": \"total_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"transformation_failed\",\n \"value\": 1\n },\n {\n \"type\": \"dedup_failed\",\n \"value\": 574\n },\n {\n \"type\": \"denorm_failed\",\n \"value\": 193\n }\n ]\n }\n ]\n}" }, { - "name": "Failure: invalid date range", + "name": "Failure: Failed to fetch metrics", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Internal Server Error", + "code": 500, "_postman_previewlanguage": "json", "header": [ { @@ -5160,15 +8481,15 @@ }, { "key": "Content-Length", - "value": "371" + "value": "317" }, { "key": "ETag", - "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + "value": "W/\"13d-xfZKi6jq4UY4l8MNfhQmydAroPc\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:58:18 GMT" + "value": "Wed, 15 Jan 2025 09:59:49 GMT" }, { "key": "Connection", @@ -5180,117 +8501,154 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:29:49+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"211553f9-ad91-4a4d-a6ba-d77d3e14b7ac\"\n },\n \"responseCode\": \"FAILED\",\n \"error\": {\n \"code\": \"FAILED_TO_FETCH_METRICS\",\n \"message\": \"Error while fetching metrics\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Analyze PII", + "item": [ + { + "name": "Data Analyze PII", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } + }, + "response": [ { - "name": "Success: JSON template with request body", + "name": "Success - Data Analyze PII", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", - "value": "application/json", - "type": "text" + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/jsontemplate1" + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, { "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "301" - }, - { - "key": "ETag", - "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" - }, - { - "key": "Date", - "value": "Tue, 14 May 2024 06:22:24 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" + "value": "application/json", + "description": "", + "type": "text" } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-16T10:27:56+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"8fd2cfe2-7912-40d1-8710-9b7da9a35bd9\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"field\": \"primary_passenger.email\",\n \"type\": \"internet\",\n \"score\": 0.5,\n \"reason\": [\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m014\",\n \"region\": \"\",\n \"score\": 1\n },\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m004\",\n \"region\": \"\",\n \"score\": 1\n }\n ]\n },\n {\n \"field\": \"primary_passenger.mobile\",\n \"type\": \"phone\",\n \"score\": 0.5,\n \"reason\": [\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m018\",\n \"region\": \"USA\",\n \"score\": 1\n },\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m005\",\n \"region\": \"\",\n \"score\": 1\n }\n ]\n }\n ]\n}" }, { - "name": "Success: SQL template query with request body", + "name": "Failure - Data Analyze PII", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } }, - "status": "OK", - "code": 200, + "status": "Internal Server Error", + "code": 500, "_postman_previewlanguage": "json", "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, { "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "594" - }, - { - "key": "ETag", - "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" - }, - { - "key": "Date", - "value": "Tue, 14 May 2024 06:03:06 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" + "value": "application/json", + "description": "", + "type": "text" } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-16T10:29:58+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75aa71f9-ef84-41b9-ae1b-64cc7afd3ffb\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"ERR_BAD_REQUEST\",\n \"message\": \"Request failed with status code 401\",\n \"trace\": \"\"\n }\n}" } ] } diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 6615ad74..abc6b51d 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -1,17 +1,23 @@ import express, { Application } from "express"; -import {router as v2Router} from "./routes/Router" -import { metricRouter } from "./routes/MetricRouter" -import { druidProxyRouter } from "./routes/DruidProxyRouter" +import { druidProxyRouter } from "./routes/DruidProxyRouter"; +import { metricRouter } from "./routes/MetricRouter"; +import { router as v2Router } from "./routes/Router"; import bodyParser from "body-parser"; -import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; -import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; +import { ResponseHandler } from "./helpers/ResponseHandler"; +import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; +import { OTelService } from "./services/otel/OTelService"; import { alertsRouter } from "./routes/AlertsRouter"; import { interceptAuditEvents } from "./services/telemetry"; +import _ from "lodash"; + + const app: Application = express(); - +// Initialisation of Open telemetry Service. +(config.otel && _.toLower(config?.otel?.enable) === "true") ? OTelService.init() : console.info("OpenTelemetry Service is Disabled"); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index b26a6906..26b9d080 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -42,8 +42,8 @@ export const config = { "connectionTimeout": process.env.kafka_connection_timeout ? parseInt(process.env.kafka_connection_timeout) : 5000 }, "topics": { // Default Kafka topics depend on type of dataset. - "createDataset": `${process.env.system_env || "local"}.ingest`, - "createMasterDataset": `${process.env.system_env || "local"}.masterdata.ingest` + "createDataset": `ingest`, + "createMasterDataset": `masterdata.ingest` } } }, @@ -98,7 +98,7 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), - "path": process.env.command_service_path || "/system/v1/dataset/command" + "paths": JSON.parse(process.env.command_service_paths || '{"dataset":"/system/v1/dataset/command","connector":"/connector/v1/register","analyzePII":"/system/data/v1/analyze/pii"}') }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", @@ -115,4 +115,14 @@ export const config = { }, "user_token_public_key": process.env.user_token_public_key || "", "is_RBAC_enabled": process.env.is_rbac_enabled || "false", + "otel": { + "enable": process.env.otel_enable || "false", + "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" + }, + "storage_types": process.env.storage_types || '{"lake_house":true,"realtime_store":true}', + "data_observability": { + "default_freshness_threshold": process.env.default_freshness_threshold ? parseInt(process.env.default_freshness_threshold) : 5, // in minutes + "data_out_query_time_period": process.env.data_out_query_time_period || "1d", + "default_query_time_period": process.env.default_query_time_period ? parseInt(process.env.default_query_time_period) : 7, // in days + } } diff --git a/api-service/src/configs/DatasetConfigDefault.ts b/api-service/src/configs/DatasetConfigDefault.ts index 27bdeda0..5c1802dc 100644 --- a/api-service/src/configs/DatasetConfigDefault.ts +++ b/api-service/src/configs/DatasetConfigDefault.ts @@ -17,7 +17,7 @@ export const defaultDatasetConfig = { } }, "dedup_config": { - "drop_duplicates": true, + "drop_duplicates": false, "dedup_key": "id", "dedup_period": 604800, // 7 days }, diff --git a/api-service/src/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts index cfc4d1cf..52d71a85 100644 --- a/api-service/src/configs/IngestionConfig.ts +++ b/api-service/src/configs/IngestionConfig.ts @@ -84,12 +84,12 @@ export const rawIngestionSpecDefaults = { "flattenSpec": [ { "type": "path", - "expr": "$.obsrv_meta.source.connector", + "expr": "$.obsrv_meta.source.entry_source", "name": "obsrv.meta.source.connector" }, { "type": "path", - "expr": "$.obsrv_meta.source.connectorInstance", + "expr": "$.obsrv_meta.source.id", "name": "obsrv.meta.source.id" } ], diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index fc77fc77..d07b01b2 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -5,11 +5,14 @@ import { v4 } from "uuid"; const commandHost = _.get(config, ["command_service_config", "host"]) const commandPort = _.get(config, ["command_service_config", "port"]) -const commandPath = _.get(config, ["command_service_config", "path"]) +const commandPaths = _.get(config, ["command_service_config", "paths"]) +const datasetPath = _.get(commandPaths, ["dataset"]) +const connectorRegisterPath = _.get(commandPaths, ["connector"]) +const analyzePIIPath = _.get(commandPaths, ["analyzePII"]) export const commandHttpService = axios.create({ baseURL: `${commandHost}:${commandPort}`, headers: { "Content-Type": "application/json" } }); -export const executeCommand = async (id: string, command: string) => { +export const executeCommand = async (id: string, command: string, userToken: string) => { const payload = { "id": v4(), "data": { @@ -17,9 +20,13 @@ export const executeCommand = async (id: string, command: string) => { "command": command } } - return commandHttpService.post(commandPath, payload) + return commandHttpService.post(datasetPath, payload, { headers: { Authorization: userToken }}) } -export const registerConnector = async (requestBody: any) => { - return commandHttpService.post("/connector/v1/register", requestBody) +export const registerConnector = async (requestBody: any, userToken: string) => { + return commandHttpService.post(connectorRegisterPath, requestBody, { headers: { Authorization: userToken }}) +} + +export const detectPII = async (requestBody: any, userToken: string) => { + return commandHttpService.post(analyzePIIPath, requestBody, { headers: { Authorization: userToken }}) } \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts index 4844ac7e..354577b6 100644 --- a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -22,7 +22,8 @@ const connectorRegisterController = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await registerConnector(payload); + const userToken = req.get('authorization') as string; + const registryResponse = await registerConnector(payload, userToken); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { @@ -118,4 +119,4 @@ const extractFileNameFromPath = (filePath: string): string[] => { return filePath.match(regex) || []; }; -export default connectorRegisterController; +export default connectorRegisterController; \ No newline at end of file diff --git a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts new file mode 100644 index 00000000..7e4a3af8 --- /dev/null +++ b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts @@ -0,0 +1,26 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import * as _ from "lodash"; +import logger from "../../logger"; +import { detectPII } from "../../connections/commandServiceConnection"; + +const code = "FAILED_TO_DETECT_PII"; +export const dataAnalyzePII = async (req: Request, res: Response) => { + const apiId = _.get(req, 'id') + try { + const userToken = req.get('authorization') as string; + const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'request']), userToken); + logger.info({apiId , message: `Detected PII successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: _.get(piiSuggestionsResponse, ["data", "result"]) }); + } catch (error: any) { + const errMessage = _.get(error, "response.data.detail") + logger.error(error, apiId, code); + let errorMessage = error; + const statusCode = _.get(error, "status") + if (!statusCode || statusCode == 500) { + errorMessage = { code, message: errMessage || "Failed to detect pii" } + } + ResponseHandler.errorResponse(errorMessage, req, res); + } +} \ No newline at end of file diff --git a/api-service/src/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/controllers/DataExhaust/DataExhaustController.ts index ceeb3358..858cf233 100644 --- a/api-service/src/controllers/DataExhaust/DataExhaustController.ts +++ b/api-service/src/controllers/DataExhaust/DataExhaustController.ts @@ -11,7 +11,7 @@ import { cloudProvider } from "../../services/CloudServices"; export const dataExhaust = async (req: Request, res: Response) => { const { params } = req; - const { datasetId } = params; + const { dataset_id } = params; const { type }: any = req.query; const momentFormat = "YYYY-MM-DD"; @@ -28,12 +28,12 @@ export const dataExhaust = async (req: Request, res: Response) => { return resData || {}; } - if (type && config.cloud_config.exclude_exhaust_types.includes(datasetId)) { + if (type && config.cloud_config.exclude_exhaust_types.includes(dataset_id)) { return ResponseHandler.errorResponse({ statusCode: 404, message: "Record not found", errCode: httpStatus["404_NAME"] }, req, res) } - const datasetRecord = await verifyDatasetExists(datasetId); + const datasetRecord = await verifyDatasetExists(dataset_id); if (datasetRecord === null) { - logger.error(`Dataset with ${datasetId} not found in live table`) + logger.error(`Dataset with ${dataset_id} not found in live table`) return ResponseHandler.errorResponse({ statusCode: 404, message: "Record not found", errCode: httpStatus["404_NAME"] }, req, res) } const dateRange = getDateRange(req); @@ -47,7 +47,7 @@ export const dataExhaust = async (req: Request, res: Response) => { return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid date range! make sure your range cannot be more than ${config.cloud_config.maxQueryDateRange} days`, errCode: "BAD_REQUEST" }, req, res) } - const resData: any = await getFromStorage(type, dateRange, datasetId); + const resData: any = await getFromStorage(type, dateRange, dataset_id); if (_.isEmpty(resData.files)) { logger.error("Date range provided does not have any backup files") return ResponseHandler.errorResponse({ statusCode: 404, message: "Date range provided does not have any backup files", errCode: "NOT_FOUND" }, req, res); diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 46f829e9..39090dda 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -27,7 +27,7 @@ const apiId = "api.data.in"; const dataIn = async (req: Request, res: Response) => { const requestBody = req.body; - const datasetId = req.params.datasetId.trim(); + const datasetId = req.params.dataset_id.trim(); const isValidSchema = schemaValidation(requestBody, validationSchema) if (!isValidSchema?.isValid) { @@ -39,34 +39,34 @@ const dataIn = async (req: Request, res: Response) => { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); } - const { entry_topic, dataset_config, api_version } = dataset + const { entry_topic, dataset_config, extraction_config, api_version } = dataset const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic if (!entryTopic) { logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); } - await send(addMetadataToEvents(datasetId, requestBody), entryTopic) + await send(addMetadataToEvents(datasetId, requestBody, extraction_config), entryTopic) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); } -const addMetadataToEvents = (datasetId: string, payload: any) => { +const addMetadataToEvents = (datasetId: string, payload: any, extraction_config: any) => { const validData = _.get(payload, "data"); const now = Date.now(); const mid = _.get(payload, "params.msgid"); const source = { id: "api.data.in", version: config?.version, entry_source: "api" }; const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source }; if (Array.isArray(validData)) { - const payloadRef = validData.map((event: any) => { - const payload = { - event, + const extraction_key: string = _.get(extraction_config, "extraction_key", 'events'); + const dedup_key: string = _.get(extraction_config, "dedup_config.dedup_key", 'id'); + const payload: any = { "obsrv_meta": obsrvMeta, "dataset": datasetId, "msgid": mid - } + }; + payload[extraction_key] = validData; + payload[dedup_key] = mid return payload; - }) - return payloadRef; } else { return ({ diff --git a/api-service/src/controllers/DataMetrics/DataMetricsController.ts b/api-service/src/controllers/DataMetrics/DataMetricsController.ts new file mode 100644 index 00000000..98314878 --- /dev/null +++ b/api-service/src/controllers/DataMetrics/DataMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DataMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") +} + +const dataMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, "data"); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default dataMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json b/api-service/src/controllers/DataMetrics/DataMetricsValidationSchema.json similarity index 100% rename from api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json rename to api-service/src/controllers/DataMetrics/DataMetricsValidationSchema.json diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts index dd6f93b6..fa5eee51 100644 --- a/api-service/src/controllers/DataOut/DataOutController.ts +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -9,7 +9,7 @@ import { executeNativeQuery, executeSqlQuery } from "../../connections/druidConn export const apiId = "api.data.out"; const dataOut = async (req: Request, res: Response) => { - const datasetId = req.params?.datasetId; + const datasetId = req.params?.dataset_id; const requestBody = req.body; const msgid = _.get(req, "body.params.msgid"); const isValidSchema = schemaValidation(requestBody, validationSchema); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 9015e9c7..7c53f538 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -4,7 +4,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import * as _ from "lodash"; import { schemaValidation } from "../../services/ValidationService"; import validationSchema from "./RequestValidationSchema.json"; -import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import { datasetService, getLiveDatasetConfigs, validateStorageSupport } from "../../services/DatasetService"; import { updateRecords } from "./DatasetCopyHelper"; import { obsrvError } from "../../types/ObsrvError"; @@ -40,6 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); + validateStorageSupport(dataset); const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); _.set(dataset, "updated_by", userID); diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index bb96dac2..d91da0f5 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -1,7 +1,7 @@ import _ from "lodash"; import { Request, Response } from "express"; import httpStatus from "http-status"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; @@ -28,6 +28,7 @@ const validateRequest = async (req: Request) => { throw obsrvError(datasetId, "DATASET_DUPLICATE_DENORM_KEY", "Duplicate denorm output fields found.", "BAD_REQUEST", 400, undefined, {duplicateKeys: duplicateDenormKeys}) } + validateStorageSupport(_.get(req, ["body", "request"])) } const datasetCreate = async (req: Request, res: Response) => { diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index e390d8bd..de326bb9 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import _ from "lodash"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { datasetImportValidation, migrateExportedDatasetV1 } from "./DatasetImportHelper"; import { obsrvError } from "../../types/ObsrvError"; @@ -15,18 +15,23 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } + const userID = (req as any)?.userID; + _.set(datasetPayload, "created_by", userID); + _.set(datasetPayload, "updated_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await importDataset(updatedDataset, overwrite); + validateStorageSupport(updatedDataset); + const dataset = await importDataset(updatedDataset, overwrite, userID); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const importDataset = async (dataset: Record, overwrite: string | any) => { +const importDataset = async (dataset: Record, overwrite: string | any, userID : string) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { + _.set(dataset, "updated_by", userID); const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts index 82acbf62..6cad65dc 100644 --- a/api-service/src/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -37,7 +37,7 @@ const listDatasets = async (request: Record): Promise { - if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") -} - +const apiId = "api.dataset.metrics"; const datasetMetrics = async (req: Request, res: Response) => { - const isValidSchema = schemaValidation(req.body, vaidationSchema); + const msgid = _.get(req, "body.params.msgid"); + const requestBody = req.body; + const dataset_id = _.get(req, "body.request.dataset_id"); + const timePeriod = _.get(req, "body.request.query_time_period") || config?.data_observability?.default_query_time_period; + + const { category }: any = req.body.request; + const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds + const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; + const endDate = dayjs().add(1, 'day').format(dateFormat); + const startDate = dayjs(endDate).subtract(timePeriod, 'day').format(dateFormat); + const intervals = `${startDate}/${endDate}`; + const isValidSchema = schemaValidation(requestBody, validationSchema); + const results = []; + if (!isValidSchema?.isValid) { - logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) - throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + logger.error({ apiId, datasetId: dataset_id, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); } - const { query } = req.body || {}; - const endpoint = query.url; - if (_.startsWith(endpoint, "/prom")) { - query.url = getBaseUrl(endpoint) - const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; - const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) - const data = _.get(apiResponse, "data"); - return res.json(data); + + const dataset = await datasetService.getDataset(dataset_id, ["id"], true) + if (!dataset) { + logger.error({ apiId, message: `Dataset with id ${dataset_id} not found in live table`, code: "DATASET_NOT_FOUND" }) + return ResponseHandler.errorResponse({ message: `Dataset with id ${dataset_id} not found in live table`, code: "DATASET_NOT_FOUND", statusCode: 404, errCode: "NOT_FOUND" }, req, res); } - else { - const query = _.get(req, ["body", "query", "body", "query"]); - const response = await executeNativeQuery(query); - ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + + try { + if (!category || category.includes("data_freshness")) { + const dataFreshnessResult = await getDataFreshness(dataset_id, intervals, defaultThreshold); + results.push(dataFreshnessResult); + } + + if (!category || category.includes("data_observability")) { + const dataObservabilityResult = await getDataObservability(dataset_id, intervals); + results.push(dataObservabilityResult); + } + + if (!category || category.includes("data_volume")) { + const dataVolumeResult = await getDataVolume(dataset_id, timePeriod, dateFormat); + results.push(dataVolumeResult); + } + + if (!category || category.includes("data_lineage")) { + const dataLineageResult = await getDataLineage(dataset_id, intervals); + results.push(dataLineageResult); + } + + if (!category || category.includes("connectors")) { + const connectorsResult = await getConnectors(dataset_id, intervals); + results.push(connectorsResult); + } + + if (!category || category.includes("data_quality")) { + const connectorsResult = await getDataQuality(dataset_id, intervals); + results.push(connectorsResult); + } + + logger.info({ apiId, msgid, requestBody, datasetId: dataset_id, message: "Metrics fetched successfully" }) + return ResponseHandler.successResponse(req, res, { status: 200, data: results }); + } + catch (error: any) { + logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS", error }); + return ResponseHandler.errorResponse({ message: "Error while fetching metrics", statusCode: 500, errCode: "FAILED", code: "FAILED_TO_FETCH_METRICS" }, req, res); + } + } export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/queries.ts b/api-service/src/controllers/DatasetMetrics/queries.ts new file mode 100644 index 00000000..ae02e1ac --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/queries.ts @@ -0,0 +1,402 @@ +import dayjs from "dayjs"; + +export const processingTimeQuery = (intervals: string, dataset_id: string) => ({ + query: { + queryType: "groupBy", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "UTC" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "processing_time", fieldName: "total_processing_time" }, + { type: "longSum", name: "count", fieldName: "count" } + ], + postAggregations: [ + { + type: "expression", + name: "average_processing_time", + expression: "case_searched((count > 0),(processing_time/count),0)" + } + ] + } +}); + +export const totalEventsQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "longSum", + name: "total_events_count", + fieldName: "count" + } + ] +}); + +export const totalFailedEventsQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "total_failed_events", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "validator" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "total_failed_events" + } + ] +}); + +export const generateTimeseriesQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "UTC" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] +}); + +export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "UTC" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] +}); + +export const dataLineageSuccessQuery = (intervals: string, dataset_id: string, column: string, value: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: column, + matchValueType: "STRING", + matchValue: value + }, + { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + } + ] + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "longSum", + name: "count", + fieldName: "count" + } + ] +}); + +export const generateTransformationFailedQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_id", + matchValueType: "STRING", + matchValue: "TransformerJob" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "count" + } + ] +}); + +export const generateDedupFailedQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "dedup" + }, + { + type: "equals", + column: "error_type", + matchValueType: "STRING", + matchValue: "DedupFailed" + } + ] + }, + name: "count" + } + ] +}); + +export const generateDenormFailedQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "denorm" + }, + { + type: "equals", + column: "error_type", + matchValueType: "STRING", + matchValue: "DenormDataNotFound" + } + ] + }, + name: "count" + } + ] +}); + +export const generateConnectorQuery = (intervals: string, dataset_id: string) => ({ + queryType: "topN", + dataSource: { + type: "table", + name: "system-events" + }, + dimension: { + type: "default", + dimension: "ctx_source_connector", + outputName: "name", + outputType: "STRING" + }, + metric: { + type: "dimension", + ordering: { + type: "lexicographic" + } + }, + threshold: 1001, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "longSum", + name: "count", + fieldName: "count" + } + ] +}); + +export const generateTotalQueryCallsQuery = (time_period: string) => ({ + end: dayjs().unix(), + query: `sum(sum_over_time(node_total_api_calls{entity="data-out"}[${time_period}]))`, + step: `${time_period}`, + start: dayjs().subtract(1, 'day').unix() +}); + +export const generateDatasetQueryCallsQuery = (dataset: string, time_period: string) => ({ + end: dayjs().unix(), + step: `${time_period}`, + query: `sum(sum_over_time(node_total_api_calls{dataset_id="${dataset}",entity="data-out"}[${time_period}]))`, + start: dayjs().subtract(1, 'day').unix(), +}); \ No newline at end of file diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index 68b272f1..892db86b 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -18,7 +18,7 @@ const validateRequest = async (req: Request) => { if (!isRequestValid.isValid) { throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) } - const datasetId = _.get(req, ["params", "datasetId"]) + const datasetId = _.get(req, ["params", "dataset_id"]) const isDataSetExists = await datasetService.checkDatasetExists(datasetId); if (!isDataSetExists) { throw obsrvError(datasetId, "DATASET_NOT_FOUND", `Dataset not exists with id:${datasetId}`, httpStatus[httpStatus.NOT_FOUND], 404) @@ -28,13 +28,13 @@ const validateRequest = async (req: Request) => { const datasetReset = async (req: Request, res: Response) => { const category = _.get(req, ["body", "request", "category"]); - const datasetId = _.get(req, ["params"," datasetId"]); - + const datasetId = _.get(req, ["params"," dataset_id"]); + const userToken = req.get('authorization') as string; await validateRequest(req); if (category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() if (pipeLineStatus == HealthStatus.UnHealthy) { - await restartPipeline({ "dataset": { "dataset_id": datasetId } }) + await restartPipeline({ "dataset": { "dataset_id": datasetId } }, userToken) } } else if (category == "query") { const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 0197d39e..e09fa5b8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -1,7 +1,7 @@ import { Request, Response } from "express"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import StatusTransitionSchema from "./RequestValidationSchema.json"; import ReadyToPublishSchema from "./ReadyToPublishSchema.json" @@ -50,6 +50,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); + const userToken = req.get('authorization') as string; validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) @@ -64,10 +65,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userID); + await publishDataset(dataset, userID, userToken); break; case "Retire": - await retireDataset(dataset, userID); + await retireDataset(dataset, userID, userToken); break; default: throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) @@ -88,6 +89,7 @@ const deleteDataset = async (dataset: Record) => { const readyForPublish = async (dataset: Record, updated_by: any) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + validateStorageSupport(draftDataset); let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); @@ -133,14 +135,15 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userID: any) => { +const publishDataset = async (dataset: Record, userID: any, userToken: string) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record + validateStorageSupport(draftDataset); _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) - await datasetService.publishDataset(draftDataset) + await datasetService.publishDataset(draftDataset, userToken); } const validateAndUpdateDenormConfig = async (draftDataset: Record) => { @@ -225,11 +228,11 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record, updated_by: any) => { +const retireDataset = async (dataset: Record, updated_by: any, userToken: string) => { await canRetireIfMasterDataset(dataset); await datasetService.retireDataset(dataset, updated_by); - await restartPipeline(dataset); + await restartPipeline(dataset, userToken); } @@ -252,8 +255,8 @@ const canRetireIfMasterDataset = async (dataset: Record) => { } } -export const restartPipeline = async (dataset: Record) => { - return executeCommand(dataset.id, "RESTART_PIPELINE") +export const restartPipeline = async (dataset: Record, userToken: string) => { + return executeCommand(dataset.id, "RESTART_PIPELINE", userToken) } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index e9fea051..c1ef0324 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -5,7 +5,7 @@ import Model from "sequelize/types/model"; import { DatasetStatus } from "../../types/DatasetModels"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { cipherService } from "../../services/CipherService"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; import { obsrvError } from "../../types/ObsrvError"; @@ -30,6 +30,7 @@ const validateRequest = async (req: Request) => { throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) } + validateStorageSupport(_.get(req, ["body", "request"])) } const validateDataset = (dataset: Record | null, req: Request) => { diff --git a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts index dc1f0863..470a479b 100644 --- a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts +++ b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts @@ -5,6 +5,8 @@ import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; import { druidHttpService } from "../../connections/druidConnection"; +import { getDatasourceList } from "../../services/DatasourceService"; +import { AxiosResponse } from "axios"; const apiId = "api.obsrv.data.sql-query"; const errorCode = "SQL_QUERY_FAILURE" @@ -25,10 +27,16 @@ export const sqlQuery = async (req: Request, res: Response) => { } as ErrorObject, req, res); } - const result = await druidHttpService.post(`${config.query_api.druid.sql_query_path}`, req.body, { - headers: { Authorization: authorization }, - }); - + const query = req.body.query as string; + let result: AxiosResponse; + if (isTableSchemaQuery(query)) { + const dataSources = await fetchDruidDataSources(); + result = createMockAxiosResponse(dataSources); + } else { + result = await druidHttpService.post(`${config.query_api.druid.sql_query_path}`, req.body, { + headers: { Authorization: authorization }, + }); + } logger.info({ messsge: "Successfully fetched data using sql query", apiId, resmsgid }) ResponseHandler.flatResponse(req, res, result) } catch (error: any) { @@ -37,4 +45,37 @@ export const sqlQuery = async (req: Request, res: Response) => { logger.error(error, apiId, code, resmsgid) ResponseHandler.errorResponse(errorMessage, req, res); } -} \ No newline at end of file +} + +const fetchDruidDataSources = async (): Promise<{ TABLE_NAME: string }[]> => { + try { + const dataSources = await getDatasourceList(); + return dataSources + .filter((ds: any) => ds.type === "druid") + .map((ds: any) => ({ TABLE_NAME: ds.dataValues.datasource_ref })); + } catch (error) { + logger.error({ message: "Failed to fetch Druid data sources", error }); + throw new Error("Failed to fetch Druid data sources"); + } +}; + +const isTableSchemaQuery = (sqlQuery?: string): boolean => { + return ( + sqlQuery + ?.trim() + .replace(/\s+/g, " ") + .toUpperCase() === + "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'DRUID'" + ); + }; + + +const createMockAxiosResponse = (data: any): AxiosResponse => { + return { + data, + status: 200, + statusText: "OK", + headers: {}, + config: {}, + } as AxiosResponse; +}; diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 10fda3e2..bcc9bcd2 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -77,7 +77,7 @@ const checkAccess = (decoded: any, action: string, req: Request, res: Response) const basicToken = (token: string, req: Request, res: Response, next: NextFunction) => { try { - const decoded = jwt.verify(token, config.user_token_public_key); + const decoded = jwt.verify(token, config.user_token_public_key, { algorithms: ['RS256'] }); if (!decoded || !_.isObject(decoded)) { return errorHandler(401, "Token verification failed or invalid token", req, res); @@ -125,8 +125,8 @@ export default { (req as any).userID = "SYSTEM"; return next(); } - - const token = req.get("x-user-token"); + const authHeader = req.headers['authorization']; + const token = authHeader && authHeader.split(' ')[1]; if (!token) { return errorHandler(401, "No token provided", req, res); } diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index 400e218f..1e4aca1c 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -4,6 +4,7 @@ "api.datasets.list", "api.datasets.read", "api.datasets.export", + "api.dataset.health", "api.data.out", "api.data.exhaust", "api.alert.list", @@ -12,7 +13,8 @@ "api.alert.silence.list", "api.alert.silence.get", "api.alert.notification.list", - "api.alert.notification.get" + "api.alert.notification.get", + "api.dataset.metrics" ], "restricted_dataset_api": [ "api.datasets.reset", @@ -46,11 +48,11 @@ "api.datasets.update", "api.datasets.import", "api.datasets.copy", - "api.dataset.health", "api.datasets.dataschema" ], "data": [ - "api.data.in" + "api.data.in", + "api.data.analyze.pii" ], "queryTemplate": [ "api.query.template.create", @@ -92,7 +94,8 @@ "schema", "file", "connector", - "sqlQuery" + "sqlQuery", + "data" ], "dataset_manager": [ "general_access", @@ -104,7 +107,8 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "data" ], "admin": [ "general_access", @@ -116,7 +120,12 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "alert", + "metric", + "silence", + "notificationChannel", + "data" ], "operations_admin": [ "alert", diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index fdf73315..ac45710c 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -20,4 +20,5 @@ druidProxyRouter.post(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.n druidProxyRouter.get(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet); druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) +druidProxyRouter.get(/\/druid\/coordinator(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({entity: Entity.DruidProxy}), wrapperService.forwardNativeGetDatasource) \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1e39d942..f9cc7905 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,19 +29,21 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; +import dataMetrics from "../controllers/DataMetrics/DataMetricsController"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; +import { dataAnalyzePII } from "../controllers/DataAnalyzePII/DataAnalyzePIIController"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/data/in/:dataset_id", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:dataset_id", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.get("/data/exhaust/:dataset_id", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); @@ -52,7 +54,7 @@ router.post("/template/query/:templateId", setDataToRequestObject("api.query.tem router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); +router.post("/datasets/reset/:dataset_id", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); @@ -60,7 +62,8 @@ router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), o router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); +router.post("/data/analyze/pii", setDataToRequestObject("api.data.analyze.pii"), onRequest({ entity: Entity.Management }),checkRBAC.handler(), dataAnalyzePII); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); -router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) - +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), dataMetrics) +router.post("/dataset/metrics", setDataToRequestObject("api.dataset.metrics"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetMetrics); diff --git a/api-service/src/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts index d83b65b1..53a6679d 100644 --- a/api-service/src/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -16,10 +16,14 @@ export class AWSStorageService implements ICloudService { const accessKeyId = _.get(config, "identity") const secretAccessKey = _.get(config, "credential") const endpoint = _.get(config, "endpoint") + const s3ForcePathStyle = _.get(config, "s3ForcePathStyle") const configuration: any = { region, credentials: { accessKeyId, secretAccessKey } } if(endpoint) { configuration.endpoint = endpoint; } + if (s3ForcePathStyle) { + configuration.forcePathStyle = s3ForcePathStyle; + } try { this.client = new S3Client(configuration); } diff --git a/api-service/src/services/DatasetMetricsService.ts b/api-service/src/services/DatasetMetricsService.ts new file mode 100644 index 00000000..d22bee3c --- /dev/null +++ b/api-service/src/services/DatasetMetricsService.ts @@ -0,0 +1,235 @@ +import axios from "axios"; +import dayjs from "dayjs"; +import _ from "lodash"; +import { config } from "../configs/Config"; +import { dataLineageSuccessQuery, generateConnectorQuery, generateDatasetQueryCallsQuery, generateDedupFailedQuery, generateDenormFailedQuery, generateTimeseriesQuery, generateTimeseriesQueryEventsPerHour, generateTotalQueryCallsQuery, generateTransformationFailedQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "../controllers/DatasetMetrics/queries"; +const druidPort = _.get(config, "query_api.druid.port"); +const druidHost = _.get(config, "query_api.druid.host"); +const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +const prometheusEndpoint = `${config.query_api.prometheus.url}/api/v1/query_range`; + +export const getDataFreshness = async (dataset_id: string, intervals: string, defaultThreshold: number) => { + const queryPayload = processingTimeQuery(intervals, dataset_id); + const druidResponse = await axios.post(nativeQueryEndpoint, queryPayload?.query); + const avgProcessingTime = _.get(druidResponse, "data[0].average_processing_time", 0); + const freshnessStatus = avgProcessingTime < defaultThreshold ? "Healthy" : "Unhealthy"; + + return { + category: "data_freshness", + status: freshnessStatus, + components: [ + { + type: "average_time_difference_in_min", + threshold: defaultThreshold / 60000, // convert to minutes + value: avgProcessingTime / 60000, + status: freshnessStatus + }, + { + type: "freshness_query_time_in_min", + threshold: 10, + value: avgProcessingTime / 60000, // convert to minutes + status: freshnessStatus + } + ] + }; +}; + +export const getDataObservability = async (dataset_id: string, intervals: string) => { + const totalEventsPayload = totalEventsQuery(intervals, dataset_id); + const totalFailedEventsPayload = totalFailedEventsQuery(intervals, dataset_id); + const totalQueryCalls = generateTotalQueryCallsQuery(config?.data_observability?.data_out_query_time_period); + const totalQueryCallsAtDatasetLevel = generateDatasetQueryCallsQuery(dataset_id, config?.data_observability?.data_out_query_time_period); + + const [totalEventsResponse, totalFailedEventsResponse, totalApiCallsResponse, totalCallsAtDatasetLevelResponse] = await Promise.all([ + axios.post(nativeQueryEndpoint, totalEventsPayload), + axios.post(nativeQueryEndpoint, totalFailedEventsPayload), + axios.request({ url: prometheusEndpoint, method: "GET", params: totalQueryCalls }), + axios.request({ url: prometheusEndpoint, method: "GET", params: totalQueryCallsAtDatasetLevel }) + ]); + + const totalApiCalls = _.map(_.get(totalApiCallsResponse, 'data.data.result'), payload => { + return _.floor(_.get(payload, 'values[0][1]'), 3) || 0 + }) + + const totalApiCallsAtDatasetLevel = _.map(_.get(totalCallsAtDatasetLevelResponse, 'data.data.result'), payload => { + return _.floor(_.get(payload, 'values[0][1]'), 3) || 0 + }) + + const importanceScore = (totalApiCallsAtDatasetLevel[0] / totalApiCalls[0]) * 100; + + const totalEventsCount = _.get(totalEventsResponse, "data[0].result.total_events_count") || 0; + const totalFailedEventsCount = _.get(totalFailedEventsResponse, "data[0].result.total_failed_events") || 0; + let failurePercentage = 0; + if (totalEventsCount > 0) { + failurePercentage = (totalFailedEventsCount / totalEventsCount) * 100; + } + const observabilityStatus = failurePercentage > 5 ? "Unhealthy" : "Healthy"; + + return { + category: "data_observability", + status: observabilityStatus, + components: [ + { + type: "data_observability_health", + status: observabilityStatus + }, + { + type: "failure_percentage", + value: failurePercentage + }, + { + type: "threshold_percentage", + value: 5 + }, + { + type: "importance_score", + value: importanceScore || 0 + } + ] + }; +}; + +export const getDataVolume = async (dataset_id: string, volume_by_days: number, dateFormat: string) => { + const currentHourIntervals = dayjs().subtract(1, "hour").startOf("hour").toISOString() + "/" + dayjs().startOf("hour").toISOString(); + const currentDayIntervals = dayjs().subtract(1, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); + const currentWeekIntervals = dayjs().subtract(1, 'week').startOf('week').format(dateFormat) + '/' + dayjs().endOf('week').format(dateFormat); + const previousHourIntervals = dayjs().subtract(2, "hour").startOf("hour").toISOString() + '/' + dayjs().startOf("hour").toISOString(); + const previousDayIntervals = dayjs().subtract(2, 'day').startOf('day').format(dateFormat) + '/' + dayjs().subtract(1, 'day').endOf('day').format(dateFormat); + const previousWeekIntervals = dayjs().subtract(2, 'week').startOf('week').format(dateFormat) + '/' + dayjs().subtract(1, 'week').endOf('week').format(dateFormat); + const nDaysIntervals = dayjs().subtract(volume_by_days, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); + + const currentHourPayload = generateTimeseriesQueryEventsPerHour(currentHourIntervals, dataset_id); + const currentDayPayload = generateTimeseriesQuery(currentDayIntervals, dataset_id); + const currentWeekPayload = generateTimeseriesQuery(currentWeekIntervals, dataset_id); + const previousHourPayload = generateTimeseriesQueryEventsPerHour(previousHourIntervals, dataset_id); + const previousDayPayload = generateTimeseriesQuery(previousDayIntervals, dataset_id); + const previousWeekPayload = generateTimeseriesQuery(previousWeekIntervals, dataset_id); + const nDaysPayload = generateTimeseriesQuery(nDaysIntervals, dataset_id); + const [ + currentHourResponse, currentDayResponse, currentWeekResponse, + previousHourResponse, previousDayResponse, previousWeekResponse, + nDaysResponse + ] = await Promise.all([ + axios.post(nativeQueryEndpoint, currentHourPayload), + axios.post(nativeQueryEndpoint, currentDayPayload), + axios.post(nativeQueryEndpoint, currentWeekPayload), + axios.post(nativeQueryEndpoint, previousHourPayload), + axios.post(nativeQueryEndpoint, previousDayPayload), + axios.post(nativeQueryEndpoint, previousWeekPayload), + axios.post(nativeQueryEndpoint, nDaysPayload) + ]); + const currentHourCount = _.get(currentHourResponse, "data[0].result.count") || 0; + const currentDayCount = _.get(currentDayResponse, "data[0].result.count") || 0; + const currentWeekCount = _.get(currentWeekResponse, "data[0].result.count") || 0; + const previousHourCount = _.get(previousHourResponse, "data[0].result.count") || 0; + const previousDayCount = _.get(previousDayResponse, "data[0].result.count") || 0; + const previousWeekCount = _.get(previousWeekResponse, "data[0].result.count") || 0; + const nDaysCount = _.get(nDaysResponse, "data[0].result.count") || 0; + + const volumePercentageByHour = previousHourCount > 0 ? ((currentHourCount - previousHourCount) / previousHourCount) * 100 : 0; + const volumePercentageByDay = previousDayCount > 0 ? ((currentDayCount - previousDayCount) / previousDayCount) * 100 : 0; + const volumePercentageByWeek = previousWeekCount > 0 ? ((currentWeekCount - previousWeekCount) / previousWeekCount) * 100 : 0; + + return { + category: "data_volume", + components: [ + { type: "events_per_hour", value: currentHourCount }, + { type: "events_per_day", value: currentDayCount }, + { type: "events_per_n_day", value: nDaysCount }, + { type: "volume_percentage_by_hour", value: volumePercentageByHour }, + { type: "volume_percentage_by_day", value: volumePercentageByDay }, + { type: "volume_percentage_by_week", value: volumePercentageByWeek }, + { type: "growth_rate_percentage", value: volumePercentageByHour } + ] + }; +}; + +export const getDataLineage = async (dataset_id: string, intervals: string) => { + const transformationSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "transformer_status", "success"); + const dedupSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "dedup_status", "success"); + const denormSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "denorm_status", "success"); + const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); + const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); + const transformationFailedPayload = generateTransformationFailedQuery(intervals, dataset_id); + const dedupFailedPayload = generateDedupFailedQuery(intervals, dataset_id); + const denormFailedPayload = generateDenormFailedQuery(intervals, dataset_id); + + const [ + transformationSuccessResponse, dedupSuccessResponse, denormSuccessResponse, + totalValidationResponse, totalValidationFailedResponse, transformationFailedResponse, dedupFailedResponse, denormFailedResponse + ] = await Promise.all([ + axios.post(nativeQueryEndpoint, transformationSuccessPayload), + axios.post(nativeQueryEndpoint, dedupSuccessPayload), + axios.post(nativeQueryEndpoint, denormSuccessPayload), + axios.post(nativeQueryEndpoint, totalValidationPayload), + axios.post(nativeQueryEndpoint, totalValidationFailedPayload), + axios.post(nativeQueryEndpoint, transformationFailedPayload), + axios.post(nativeQueryEndpoint, dedupFailedPayload), + axios.post(nativeQueryEndpoint, denormFailedPayload) + ]); + + // success at each level + const transformationSuccessCount = _.get(transformationSuccessResponse, "data[0].result.count") || 0; + const dedupSuccessCount = _.get(dedupSuccessResponse, "data[0].result.count") || 0; + const denormSuccessCount = _.get(denormSuccessResponse, "data[0].result.count") || 0; + const totalValidationCount = _.get(totalValidationResponse, "data[0].result.count") || 0; + const totalValidationFailedCount = _.get(totalValidationFailedResponse, "data[0].result.count") || 0; + const storageSuccessCount = totalValidationCount - totalValidationFailedCount; + + // failure at each level + const transformationFailedCount = _.get(transformationFailedResponse, "data[0].result.count") || 0; + const dedupFailedCount = _.get(dedupFailedResponse, "data[0].result.count") || 0; + const denormFailedCount = _.get(denormFailedResponse, "data[0].result.count") || 0; + return { + category: "data_lineage", + components: [ + { type: "transformation_success", value: transformationSuccessCount }, + { type: "dedup_success", value: dedupSuccessCount }, + { type: "denormalization_success", value: denormSuccessCount }, + { type: "total_success", value: storageSuccessCount }, + { type: "total_failed", value: totalValidationFailedCount }, + { type: "transformation_failed", value: transformationFailedCount }, + { type: "dedup_failed", value: dedupFailedCount }, + { type: "denorm_failed", value: denormFailedCount } + ] + }; +}; + + +export const getConnectors = async (dataset_id: string, intervals: string) => { + const connectorQueryPayload = generateConnectorQuery(intervals, dataset_id); + const connectorResponse = await axios.post(nativeQueryEndpoint, connectorQueryPayload); + const connectorsData = _.get(connectorResponse, "data[0].result", []); + const result = { + category: "connectors", + components: connectorsData.map((item: any) => ({ + id: item.name || "failed", + type: item.name === null ? "failed" : "success", + value: item.count + })) + }; + + return result; +}; + +export const getDataQuality = async (dataset_id: string, intervals: string) => { + const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); + const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); + const [totalValidationResponse, totalValidationFailedResponse, + ] = await Promise.all([ + axios.post(nativeQueryEndpoint, totalValidationPayload), + axios.post(nativeQueryEndpoint, totalValidationFailedPayload), + ]); + const totalValidationCount = _.get(totalValidationResponse, "data[0].result.count") || 0; + const totalValidationFailedCount = _.get(totalValidationFailedResponse, "data[0].result.count") || 0; + const storageSuccessCount = totalValidationCount - totalValidationFailedCount; + + return { + category: "data_quality", + "components": [ + { type: "incidents_failed", value: totalValidationFailedCount }, + { type: "incidents_success", value: storageSuccessCount }, + { type: "total_incidents", value: totalValidationCount } + ] + }; +} \ No newline at end of file diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index e8ce9a35..b953b47f 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -18,6 +18,7 @@ import { obsrvError } from "../types/ObsrvError"; import { druidHttpService } from "../connections/druidConnection"; import { tableGenerator } from "./TableGenerator"; import { deleteAlertByDataset, deleteMetricAliasByDataset } from "./managers"; +import { config } from "../configs/Config"; class DatasetService { @@ -302,7 +303,7 @@ class DatasetService { } } - publishDataset = async (draftDataset: Record) => { + publishDataset = async (draftDataset: Record, userToken: string) => { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() @@ -325,7 +326,7 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET", userToken); } @@ -411,4 +412,16 @@ export const getV1Connectors = async (datasetId: string) => { return modifiedV1Connectors; } +const storageTypes = JSON.parse(config.storage_types) +export const validateStorageSupport = (dataset: Record) => { + const { olap_store_enabled, lakehouse_enabled } = _.get(dataset, ["dataset_config", "indexing_config"]) || {} + const validStorageType = _.keys(storageTypes).filter(key => storageTypes[key] === true); + if (olap_store_enabled && !_.get(storageTypes, "realtime_store") === true) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "realtime_store" is not available. Please use one of the available storage types: ${validStorageType}`, "BAD_REQUEST", 400) + } + if (lakehouse_enabled && !_.get(storageTypes, "lake_house") == true) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "lake_house" is not available. Please use one of the available storage types: ${validStorageType}`, "BAD_REQUEST", 400) + } +} + export const datasetService = new DatasetService(); \ No newline at end of file diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index 812a5611..b5838900 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,16 +1,13 @@ import { Datasource } from "../models/Datasource"; -export const getDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await Datasource.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - - +export const getDatasourceList = async (datasetId?: string, raw: boolean = false) => { + const query: any = { raw }; + if (datasetId) { + query.where = { dataset_id: datasetId }; + } + + return Datasource.findAll(query); +}; diff --git a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts index 45d76c9c..ac09027e 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts @@ -43,8 +43,11 @@ export class SchemaHandler { return _.set(schema, `${absolutePath}`, { ...schema[absolutePath], ...{ - type: resolution.value, - oneof: conflict.schema.values.map(key => ({ type: key })), + type: resolution.value || _.first(conflict.schema.values), + oneof: conflict.schema.values.map(key => { + const storeFormat = _.get(dataMappingPaths, key); + return { type: _.get(DataMappings, storeFormat) } + }), } }); } @@ -89,12 +92,13 @@ export class SchemaHandler { } private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { - const types = _.get(fieldData, type); - types && types.map((item: any) => { - const storeFormat = _.get(dataMappingPaths, item.type); + const types = _.get(fieldData, type) + const propType = _.get(fieldData, "type") + if(types){ + const storeFormat = _.get(dataMappingPaths, propType); _.set(schema, `${property}.arrival_format`, _.first(storeFormat.split("."))); _.set(schema, `${property}.data_type`, _.get(DataMappings, storeFormat)); - }) + } return; } diff --git a/api-service/src/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json index e9fe2a6f..f2461a05 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaMapping.json +++ b/api-service/src/services/SchemaGenerateService/SchemaMapping.json @@ -48,7 +48,7 @@ "datasource": "long" }, "float": { - "jsonSchema": "number", + "jsonSchema": "float", "datasource": "double" }, "long": { @@ -56,7 +56,7 @@ "datasource": "long" }, "double": { - "jsonSchema": "number", + "jsonSchema": "double", "datasource": "double" }, "bigdecimal": { @@ -66,6 +66,10 @@ "epoch":{ "jsonSchema": "integer", "datasource": "long" + }, + "number":{ + "jsonSchema": "double", + "datasource": "double" } } }, diff --git a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts index fcaba4a5..176ea74a 100644 --- a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts +++ b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts @@ -2,12 +2,22 @@ import _ from "lodash" import { Conflict, ConflictTypes, Suggestion, SuggestionsTemplate } from "../../types/SchemaModel" import constants from "./Constants.json" import { SchemaSuggestionTemplate } from "./Template" +import { dataMappingPaths } from "./SchemaHandler" +import DataMappings from "./SchemaMapping.json"; export class SuggestionTemplate { public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { return _.map(sample, (value) => { - const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) + let updatedConflicts: any = {} + if (value.schema.conflicts) { + updatedConflicts = _.mapKeys(value.schema.conflicts, (value, key) => { + const storeFormat = _.get(dataMappingPaths, key); + if (!storeFormat) return key + return _.get(DataMappings, storeFormat); + }) + } + const dataTypeSuggestions = this.getSchemaMessageTemplate({ ...value.schema, ...(!_.isEmpty(updatedConflicts) && { conflicts: updatedConflicts }) }) const requiredSuggestions = this.getRequiredMessageTemplate(value.required) const formatSuggestions = this.getPropertyFormatTemplate(value.formats) return { diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index 7030ad95..a3e24d58 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -58,7 +58,7 @@ class BaseTableGenerator { const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); - _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + `['${denormField.denorm_out_field}']`)); return prop; }); dataFields.push(...transformProps); @@ -66,7 +66,7 @@ class BaseTableGenerator { } if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ - expr: "$." + tf.field_key, + expr: "$." + tf.field_key.split('.').map((fieldpart: string) => `['${fieldpart}']`).join('.'), name: tf.field_key, data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, diff --git a/api-service/src/services/WrapperService.ts b/api-service/src/services/WrapperService.ts index c38afb6d..8742f931 100644 --- a/api-service/src/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -85,6 +85,22 @@ class WrapperService { } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } }; + public forwardNativeGetDatasource = async ( + req: Request, + res: Response, + next: NextFunction + ) => { + try { + const headers = req?.headers; + const url = req?.url; + const result = await axios.get( + `${config.query_api.druid.host}:${config.query_api.druid.port}${url}`, + { headers } + ); + ResponseHandler.flatResponse(req, res, result); + } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } + }; + public nativeStatus = async ( req: Request, res: Response, diff --git a/api-service/src/services/otel/OTelService.ts b/api-service/src/services/otel/OTelService.ts new file mode 100644 index 00000000..65186cbd --- /dev/null +++ b/api-service/src/services/otel/OTelService.ts @@ -0,0 +1,158 @@ +import { Counter, diag, DiagConsoleLogger, DiagLogLevel, Meter, metrics } from '@opentelemetry/api'; +import * as logsAPI from '@opentelemetry/api-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import logger from '../../logger'; +import * as _ from "lodash"; +import { config } from "../../configs/Config"; +const collectorEndpoint = _.get(config, "otel.collector_endpoint", "http://localhost:4318"); + +export class OTelService { + private static meterProvider: MeterProvider; + private static loggerProvider: LoggerProvider; + private static tracerProvider: NodeTracerProvider; + + public static init() { + this.tracerProvider = this.createTracerProvider(collectorEndpoint); + this.meterProvider = this.createMeterProvider(collectorEndpoint); + this.loggerProvider = this.createLoggerProvider(collectorEndpoint); + + // Register the global tracer, meter, and logger providers + this.tracerProvider.register(); + this.setGlobalMeterProvider(this.meterProvider); + + logger.info("OpenTelemetry Service Initialized"); + + // Add shutdown hook + process.on('SIGTERM', async () => { + await this.tracerProvider.shutdown(); + await this.meterProvider.shutdown(); + await this.loggerProvider.shutdown(); + }); + } + + private static createTracerProvider(endpoint: string) { + const traceExporter = new OTLPTraceExporter({ + url: `${endpoint}/v1/traces`, + }); + + const tracerProvider = new NodeTracerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + tracerProvider.addSpanProcessor(new BatchSpanProcessor(traceExporter)); + + return tracerProvider; + } + + private static createMeterProvider(endpoint: string) { + const metricExporter = new OTLPMetricExporter({ + url: `${endpoint}/v1/metrics`, + }); + + const meterProvider = new MeterProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + meterProvider.addMetricReader( + new PeriodicExportingMetricReader({ + exporter: metricExporter, + exportIntervalMillis: 10000, + }) + ); + + return meterProvider; + } + + private static createLoggerProvider(endpoint: string) { + const logExporter = new OTLPLogExporter({ + url: `${endpoint}/v1/logs`, + }); + + const loggerProvider = new LoggerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + loggerProvider.addLogRecordProcessor( + new BatchLogRecordProcessor(logExporter) + ); + + return loggerProvider; + } + + // Helper method to create a Resource with service name + private static createServiceResource(serviceName: string) { + return new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: serviceName, + }); + } + + private static setGlobalMeterProvider(meterProvider: MeterProvider) { + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); + diag.info('Registering MeterProvider globally.'); + metrics.setGlobalMeterProvider(meterProvider); + } + + // Method to create a counter metric + public static createCounterMetric(name: string): Counter { + const meter = this.getMeterProvider(); // Use the updated getMeterProvider method + const counter = meter.createCounter(name, { + description: 'Counts the number of API calls', + }); + return counter; + } + + public static getMeterProvider(): Meter { + return this.meterProvider.getMeter('obsrv-api-service'); + } + + public static getLoggerProvider(): LoggerProvider { + return this.loggerProvider; + } + + public static getTracerProvider(): NodeTracerProvider { + return this.tracerProvider; + } + + // Method to record the counter metric + public static recordCounter(counter: Counter, value: number) { + counter.add(value, { + service: 'obsrv-api-service', + }); + } + + + public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { + if((config.otel && _.toLower(config?.otel?.enable) === "true")){ + const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); + + const severityMapping: Record = { + INFO: logsAPI.SeverityNumber.INFO, + WARN: logsAPI.SeverityNumber.WARN, + ERROR: logsAPI.SeverityNumber.ERROR, + }; + + const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; + + const logRecord = { + severityNumber, + severityText: severity, + body: JSON.stringify(auditLog), + attributes: { + 'log.type': logType || 'console', + ...auditLog, + }, + }; + loggerInstance.emit(logRecord); + } + } + + +} diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index 8408dfb3..7aa4db86 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -3,6 +3,7 @@ import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import {send} from "../connections/kafkaConnection" +import { OTelService } from "./otel/OTelService"; const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); @@ -50,7 +51,8 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { - send({ messages: [{ value: JSON.stringify(event) }] }, telemetryTopic).catch(console.log); + OTelService.generateOTelLog(event, 'INFO', 'audit-log'); + send(event, telemetryTopic).catch(console.log); } const transformProps = (body: Record) => { diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index d4cd25b5..1fcdf368 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -1,36 +1,38 @@ openapi: 3.0.0 info: - title: Obsrv API V2 + title: Obsrv v2 apis description: >- Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets. version: 1.0.0 servers: - - url: ":3000" + - url: http://localhost:3000 + - url: http://{{host_ip}} tags: - - name: Dataset CRUD APIs + - name: Dataset API's description: >- The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria. + - name: Connector API's - name: Data Ingest - - name: Data query - - name: Data exhaust + - name: Data Query - name: Query Templates - - name: Dataset copy - - name: Schema validator + - name: Alert Notification Channels + - name: Alert Silence + - name: Alerts Wrapper + - name: Alert Metric Alias paths: /v2/datasets/create: post: tags: - - Dataset CRUD APIs - summary: Dataset create + - Dataset API's + summary: Create Dataset description: >- This API allows you to create new datasets used by the analytical data source. requestBody: - required: true content: application/json: schema: @@ -42,8 +44,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - type: dataset + dataset_id: telemetry_record-t4 + type: event name: sb-telemetry validation_config: validate: true @@ -78,148 +80,418 @@ paths: denorm_fields: - denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry transformations_config: - - field_key: eid + - field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - dataset_config: - data_key: mid - timestamp_key: ets - files_upload_path: ["telemetry.json"] tags: - tag1 - - tag2 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '287' + ETag: + schema: + type: string + example: W/"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ" + Date: + schema: + type: string + example: Mon, 15 Jul 2024 13:14:09 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset Success + summary: 'Success: Dataset created successfullly' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-15T18:44:08+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 276c042c-0f23-4b26-9b10-6fe48bbc2d3a responseCode: OK result: - id: sb-telemetry.1 - version_key: '1713442037275' + id: telemetry_record-t4 + version_key: '1721049248930' example-1: - summary: Master Dataset Success + summary: 'Success: Master dataset created successfully' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:36:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 845076be-d9e7-4246-bb8e-07ae0ce59d1e + responseCode: OK + result: + id: telemetry_record-master + version_key: '1721099200603' + example-2: + summary: 'Success: Minimal request body' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T18:14:59+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9e207f4f-2be6-4a45-ab78-213bea272ae0 + responseCode: OK + result: + id: telemetry_events + version_key: '1721133899306' + example-3: + summary: 'Success: Dataset created successfully with all fields' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-17T18:19:53+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 505fb3bc-ae32-4f5b-a931-adec4d1d84ba responseCode: OK result: - id: sb-telemetry-master.1 - version_key: '1713442037275' + id: telemetry_record-t41 + version_key: '1721220593027' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '362' + ETag: + schema: + type: string + example: W/"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:09:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:39:00+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a07de860-dcbc-4ff6-822e-34b47635c8a3 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '337' + ETag: + schema: + type: string + example: W/"151-a7dJ9XBUyT3AXNxl1TPcraxMX08" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:07:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Dataset contains duplicate denorm out field' + summary: 'Failure: Master dataset already exists' value: - id: api.datasets.create + id: api.files.generate-url ver: v2 - ts: '2024-04-16T17:59:06+05:30' + ts: '2024-07-16T08:37:28+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 54c5b692-dc37-432e-b556-7f396d7c9e07 - responseCode: BAD_REQUEST + resmsgid: 138b796b-1b68-481a-a59d-1cb695c1adc9 + responseCode: CONFLICT + result: {} error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Duplicate denorm key found - trace: '' + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-master example-1: - summary: 'Failure: Invalid request payload provided' + summary: 'Failure: Dataset already exists' value: - id: api.datasets.create + id: api.files.generate-url ver: v2 - ts: '2024-04-16T18:00:34+05:30' + ts: '2024-07-16T08:38:05+05:30' params: status: FAILED - resmsgid: 615c1e4c-8c19-44fd-b29c-c235e7cbb5f0 - responseCode: BAD_REQUEST + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf62693c-3aa4-42ce-a5ea-4bde340740f5 + responseCode: CONFLICT + result: {} error: - code: DATASET_INVALID_INPUT - message: >- - #additionalProperties should NOT have additional - properties - trace: '' - '409': - description: Conflict + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-t4 + /v2/files/generate-url: + post: + tags: + - Dataset API's + summary: Generate Presigned URLs + description: This API generates presigned URLs to upload or download files from cloud + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.files.generate-url + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '1344' + ETag: + schema: + type: string + example: W/"540-790rZel+H/rDwgvZRxvlUmZ8Gpc" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 02:56:19 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T17:56:06+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 76612ad1-241b-4ce3-8af4-88db860697f4 - responseCode: CONFLICT - error: - code: DATASET_EXISTS - message: Dataset already exists - trace: '' - '500': - description: Internal Server Error + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '355' + ETag: + schema: + type: string + example: W/"163-9oQYJJEaBH3mJAnzDHXn2MxE848" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:03:04 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T18:02:44+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dd1c0e11-fb4c-484c-81fe-82c9e2eee053 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_CREATION_FAILURE - message: Failed to create dataset - trace: '' + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' /v2/datasets/update: patch: tags: - - Dataset CRUD APIs - summary: Dataset update + - Dataset API's + summary: Update Dataset description: >- This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or - update transformations + update transformations and connectors requestBody: - required: true content: application/json: schema: @@ -231,8 +503,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - version_key: '1717073955640' + dataset_id: telemetry_record-t4 + version_key: '1721135455988' name: sb-telemetry validation_config: validate: true @@ -242,279 +514,254 @@ paths: extraction_key: events dedup_config: drop_duplicates: true - dedup_key: id + dedup_key: ipid dedup_config: drop_duplicates: true - dedup_key: eid + dedup_key: mid data_schema: $schema: https://json-schema.org/draft/2020-12/schema type: object properties: - eid: + midpid: type: string - ets: + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: + type: string + arrival_format: text + data_type: string + sid: type: string - required: - - eid + arrival_format: text + data_type: string additionalProperties: true denorm_config: denorm_fields: - - values: + - value: denorm_key: eid denorm_out_field: userdata - dataset_id: master-telemetry action: remove - - values: + - value: denorm_key: eid denorm_out_field: edata - dataset_id: trip-record - action: add - transformation_config: - - values: - field_key: eid + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert + - value: + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - action: add - dataset_config: - data_key: eid - timestamp_key: ets - tags: - - values: - - tag1 - - tag2 action: remove - - values: - - tag3 - - tag4 - action: add + tags: [] + connectors_config: + - value: + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice + action: upsert + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '331' + ETag: + schema: + type: string + example: W/"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 13:00:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T00:16:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 4b2c0c22-f765-46f9-80af-33d0442db5ca - responseCode: OK - result: - message: Dataset is updated successfully - id: sb-telemetry.1 - version_key: '1713465973004' + examples: + example-0: + summary: 'Success: Minimal dataset update' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:30:45+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 354f1fec-0c39-42ee-a52a-49552f847c11 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134845559' + example-1: + summary: 'Success: Updated successfully' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:27:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 6d835f07-aed5-4e8b-81c2-2142cfb55c52 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134675878' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '369' + ETag: + schema: + type: string + example: W/"171-iNJoyWUecOEsXbHZwx6rld3Sr1I" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:59:21 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Invalid payload provided' + summary: 'Failure: Invalid request' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:22:12+05:30' + ts: '2024-07-16T18:29:21+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9408e137-ada8-48b8-99e9-90d5cdd35e35 + resmsgid: 7d31672b-e5c3-4a6d-afac-d9d78011bcde responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_INPUT_INVALID message: >- - #properties/request/properties/name/type should be - string - trace: '' + #properties/request/required must have required property + 'dataset_id' example-1: - summary: 'Failure: No field provided along with dataset_id' + summary: 'Failure: No fields are provided to update' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:23:06+05:30' + ts: '2024-07-16T18:32:44+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 347f465f-f100-4095-9801-38c4943380c0 + resmsgid: bf99b1e1-7694-4be0-ba5d-e347764736de responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_NO_FIELDS message: >- Provide atleast one field in addition to the dataset_id to update the dataset - trace: '' - example-2: - summary: 'Failure: Cannot update as dataset not in draft state' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:25:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: a9dfc926-c893-4ec3-82b8-bc8601d928a9 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE - message: Dataset cannot be updated as it is not in draft state - trace: '' - example-3: - summary: 'Failure: Dataset contains duplicate denorm out field' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:29:25+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 865b9bef-9b6d-467c-9065-6689126b0f42 - responseCode: BAD_REQUEST - error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Dataset contains duplicate denorm out keys:[userdata] - trace: '' - example-4: - summary: 'Failure: Dataset tags to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:32:05+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9e9ff0bc-d660-4c4e-a2b0-cb11246a6961 - responseCode: BAD_REQUEST - error: - code: DATASET_TAGS_EXISTS - message: Dataset tags already exist - trace: '' - example-5: - summary: 'Failure: Dataset transformations to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:34:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e0a5cc26-032e-4be8-81ef-c7e76867c9c1 - responseCode: BAD_REQUEST - error: - code: DATASET_TRANSFORMATIONS_EXIST - message: Dataset transformations already exists - trace: '' - example-6: - summary: 'Failure: Dataset denorm fields to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:38:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b807580-a2b5-4a44-adc4-80b35b4b6529 - responseCode: BAD_REQUEST - error: - code: DATASET_DENORM_EXISTS - message: Denorm fields already exist - trace: '' '404': description: Not Found - content: - application/json: + headers: + X-Powered-By: schema: - type: object - examples: - example-0: - summary: 'Failure: Dataset does not exists to update' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:20:39+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ebcf0a62-7c95-40f5-adf2-87ab90a40a80 - responseCode: NOT_FOUND - error: - code: DATASET_NOT_EXISTS - message: Dataset does not exists to update - trace: '' - example-1: - summary: 'Failure: Dataset tags to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:33:33+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: c29e7d0f-3eaf-4972-8804-c172db7b6f13 - responseCode: NOT_FOUND - error: - code: DATASET_TAGS_DO_NOT_EXIST - message: Dataset tags do not exist to remove - trace: '' - example-2: - summary: 'Failure: Dataset transformations to update do not exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:36:04+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b5f515b-1958-4597-b603-b4038d4e5846 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to update - trace: '' - example-3: - summary: 'Failure: Dataset transformations to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:37:20+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 69d32411-4252-4b7f-8afa-6d436928c3d3 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to remove - trace: '' - example-4: - summary: 'Failure: Dataset denorm to remove does not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:40:07+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ce78e393-2b74-46da-a70b-2abd06ad3270 - responseCode: NOT_FOUND - error: - code: DATASET_DENORM_DO_NOT_EXIST - message: Denorm fields do not exist to remove - trace: '' - '409': - description: Conflict + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '340' + ETag: + schema: + type: string + example: W/"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:58:30 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -522,20 +769,47 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-24T12:30:18+05:30' + ts: '2024-07-16T18:28:30+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: fe4a438d-441c-4497-a3ec-a0cee2058b48 - responseCode: CONFLICT + resmsgid: bf64703c-bb6b-41bf-bc1a-c85373efd925 + responseCode: NOT_FOUND + result: {} error: - code: DATASET_OUTDATED - message: >- - The dataset is outdated. Please try to fetch latest changes - of the dataset and perform the updates - trace: '' - '500': - description: Internal Server Error + code: DATASET_NOT_EXISTS + message: Dataset does not exists with id:telemetry_record-t41 + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '383' + ETag: + schema: + type: string + example: W/"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:53:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -543,578 +817,230 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:41:33+05:30' + ts: '2024-07-16T18:23:16+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 451daa85-9cb8-459a-b681-a256c6e912f0 - responseCode: INTERNAL_SERVER_ERROR + resmsgid: 02fe03f6-c4c4-48f6-9d84-a32cd52f4c13 + responseCode: CONFLICT + result: {} error: - code: DATASET_UPDATE_FAILURE - message: Failed to update dataset - trace: '' + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates /v2/datasets/read/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset get: tags: - - Dataset CRUD APIs - summary: Dataset read + - Dataset API's + summary: Read Dataset description: >- This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status - "Live". - This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, - it creates one using the live dataset and returns the dataset details. -
-
- Valid fields that user can request are - "dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date" + "Live". This API accepts the parameter mode=edit to read the draft + dataset. If a draft dataset is not found, it creates one using the live + dataset and returns the dataset details. parameters: - - name: dataset_id - example: sb_telemetry - in: path - required: true - schema: - type: string - - name: status - example: Draft - in: query - required: false - schema: - type: string - - name: fields - example: name - in: query - required: false + - name: Cookie + in: header schema: type: string + example: >- + connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4 - name: mode - example: edit in: query - required: false schema: type: string + example: edit responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '501' + ETag: + schema: + type: string + example: W/"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:08:55 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Read Draft dataset + summary: 'Success: Read live dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:43:58+05:30' + ts: '2024-07-17T17:38:55+05:30' params: status: SUCCESS - resmsgid: d02c643a-d51d-4c67-8a42-9f43e61f459e + resmsgid: 8c8a2852-54bc-43fb-b063-7f359d11930a responseCode: OK result: - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets - entry_topic: local.ingest + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} + redis_db: 54 example-1: - summary: Read Live dataset + summary: 'Success: Read draft dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:44:54+05:30' + ts: '2024-07-17T17:41:00+05:30' params: status: SUCCESS - resmsgid: 3d0efbc5-dea8-4815-a749-b9f9ae16ccfe + resmsgid: 96fd4f42-fa84-4730-bc79-d241a4e335a1 responseCode: OK result: - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts - exclude_fields: [] - entry_topic: local.ingest + timestamp_key: ets + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - version: 1 + file_upload_path: [] + exclude_fields: [] example-2: - summary: Read specific fields from the dataset + summary: 'Success: Read specific column' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:45:44+05:30' + ts: '2024-07-17T17:42:16+05:30' params: status: SUCCESS - resmsgid: da752dfe-0d88-4dd6-a6a8-d858f5960f7e + resmsgid: 02a6b03a-8bf3-4e37-8dcd-859d3e8f904e responseCode: OK result: - name: sb-telemetry - type: dataset - id: sb-telemetry.1 + name: master-test + type: master + id: master-test example-3: - summary: Read version_key from the dataset + summary: 'Success: Read version_key' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:47:48+05:30' + ts: '2024-07-17T17:45:37+05:30' params: status: SUCCESS - resmsgid: a3ad1e17-671f-4294-a834-2ee3f255c9b3 + resmsgid: 805e848a-d260-47c3-b55c-fc9b8323719e responseCode: OK result: - version_key: '1713510527662' + version_key: '1718791650227' example-4: - summary: 'Read Dataset: Create a draft dataset on edit live' + summary: >- + Success: Read from draft, if not present cerate draft from + live dataset and then read value: id: api.datasets.read ver: v2 - ts: '2024-06-18T14:44:54+05:30' + ts: '2024-07-17T17:49:28+05:30' params: status: SUCCESS - resmsgid: 9f8f6a80-f056-424a-97c4-179e2b9199f2 + resmsgid: da70e25b-6ad0-49a7-a39d-340d1d0c46a7 responseCode: OK result: - id: master-telemetry - dataset_id: master-telemetry - type: master-dataset - name: master-telemetry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - eid: - type: string - arrival_format: text - data_type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - arrival_format: text - data_type: string - updatedBy: - type: string - arrival_format: text - data_type: string - lastAccessedOn: - type: string - arrival_format: text - data_type: string - content_id: - type: string - arrival_format: text - data_type: string - oldLastAccessedOn: - type: string - arrival_format: text - data_type: string - progress: - type: number - arrival_format: number - data_type: number - previousProgress: - type: number - arrival_format: number - data_type: number - contentType: - type: string - arrival_format: text - data_type: string - resourceType: - type: string - arrival_format: text - data_type: string - objectId: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - type: - type: string - arrival_format: text - data_type: string - props: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - ver: - type: string - arrival_format: text - data_type: string - syncts: - type: number - arrival_format: number - data_type: number - '@timestamp': - type: string - arrival_format: text - data_type: string - ets: - type: number - arrival_format: number - data_type: number - context: - type: object - properties: - channel: - type: string - arrival_format: text - data_type: string - pdata: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - env: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - flags: - type: object - properties: - ex_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_validation_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_duplicate_skipped: - type: boolean - arrival_format: boolean - data_type: boolean - arrival_format: object - data_type: object - additionalProperties: true - mid: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - additionalProperties: true - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetry - dataset_config: - data_key: ver - timestamp_key: obsrv_meta.syncts - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 70 - file_upload_path: [] - configurations: - indexConfiguration: - index: - Event Arrival Time: obsrv_meta.syncts - rollupSuggestions: {} - processing: - dedupKeys: [] - dropDuplicates: - - 'Yes' - - 'No' - mergedEvent: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - type: - type: string - required: - - id - - type - eid: - type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - updatedBy: - type: string - lastAccessedOn: - type: string - content_id: - type: string - oldLastAccessedOn: - type: string - progress: - type: number - previousProgress: - type: number - contentType: - type: string - resourceType: - type: string - objectId: - type: string - required: - - oldFirstAccessedOn - - updatedBy - - lastAccessedOn - - content_id - - oldLastAccessedOn - - progress - - previousProgress - - contentType - - resourceType - - objectId - type: - type: string - props: - type: string - required: - - data - - type - - props - ver: - type: string - syncts: - type: number - '@timestamp': - type: string - ets: - type: number - context: - type: object - properties: - channel: - type: string - pdata: - type: object - properties: - id: - type: string - required: - - id - env: - type: string - required: - - channel - - pdata - - env - flags: - type: object - properties: - ex_processed: - type: boolean - pp_validation_processed: - type: boolean - pp_duplicate_skipped: - type: boolean - required: - - ex_processed - - pp_validation_processed - - pp_duplicate_skipped - mid: - type: string - type: - type: string - required: - - actor - - eid - - edata - - ver - - syncts - - '@timestamp' - - ets - - context - - flags - - mid - - type + dataset_id: sample1 + name: sample1 + type: event + status: Live tags: [] - created_by: SYSTEM - updated_by: SYSTEM - version_key: '1718702094143' - version: 1 - status: Draft + version: 2 api_version: v2 - transformations_config: [] + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + data_key: '' + timestamp_key: time + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '307' + ETag: + schema: + type: string + example: W/"133-TQ9WpmutsrDcTNkRRmbWOhUChMk" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:20:17 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1122,19 +1048,48 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:03+05:30' + ts: '2024-07-17T17:50:17+05:30' params: status: FAILED - resmsgid: 302fec39-5070-4464-b02f-56b1d0418147 + resmsgid: bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd responseCode: BAD_REQUEST + result: {} error: code: DATASET_INVALID_FIELDS message: >- The specified fields [newname] in the dataset cannot be found. - trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '304' + ETag: + schema: + type: string + example: W/"130-JL/oBB+GUHTrBp278giBHRvO71I" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:21:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1142,43 +1097,26 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:14+05:30' + ts: '2024-07-17T17:51:12+05:30' params: status: FAILED - resmsgid: cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c + resmsgid: 3aad3842-a76e-4fe8-b635-c7fef5f318f9 responseCode: NOT_FOUND + result: {} error: code: DATASET_NOT_FOUND - message: Dataset with the given dataset_id not found - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.read - ver: v2 - ts: '2024-04-19T12:46:34+05:30' - params: - status: FAILED - resmsgid: 29d82020-101d-4506-8fb8-6f3603057064 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_READ_FAILURE - message: Failed to read dataset - trace: '' + message: >- + Dataset with the given dataset_id:new_telemetry_record.1 not + found /v2/datasets/list: post: tags: - - Dataset CRUD APIs - summary: Dataset List + - Dataset API's + summary: List Datasets description: >- This API allows you to list all datasets. User can apply filters on - dataset status and type, sort the datasets as per requested order. + dataset status and type. requestBody: - required: true content: application/json: schema: @@ -1191,70 +1129,65 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: filters: - type: master-dataset + status: + - Live responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '24340' + ETag: + schema: + type: string + example: W/"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:25:36 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset list success when no filters provided + summary: 'Success: Lists all when no filters provided' value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:54:56+05:30' + ts: '2024-07-17T17:55:36+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e7a2c5c1-5aa9-4422-81a5-6f197d3b689e + resmsgid: 97efe04d-e981-493d-9ee7-a6dad6887d64 responseCode: OK result: data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1264,427 +1197,222 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_pickup_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill + data_key: userid timestamp_key: '' exclude_fields: [] entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: two.1 - dataset_id: two - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid + data_key: '' timestamp_key: ets - entry_topic: local.ingest - redis_db_host: localhost + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713509698978' - created_date: '2024-04-15T02:22:48.270Z' - updated_date: '2024-04-15T02:24:03.308Z' - transformations_config: - - field_key: eid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - field_key: cid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets + data_key: '' + timestamp_key: date + exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} - count: 6 - example-1: - summary: Dataset list success when status filter provided as array - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:55:39+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 836f853f-9ad8-4567-a9bb-74c5d3bba25d - responseCode: OK - result: - data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1694,208 +1422,139 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: beckn-test-data + name: beckn-test-data + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: telemetry_record-t4 + name: sb-telemetry + type: event + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_events + name: sb-telemetry + type: event + status: Draft + tags: [] + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_record-master + name: sb-telemetry + type: master + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 dataset_config: - data_key: skill + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: generate-schema + name: generate-schema + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' timestamp_key: '' exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' version: 1 - count: 2 - example-2: - summary: Dataset list success when status filter provided as string - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:56:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 469ef16b-4211-4b71-b063-bbd81cc9df02 - responseCode: OK - result: - data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + api_version: null dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: obsrv_meta.syncts exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1905,1453 +1564,6520 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 + - dataset_id: trip-test + name: trip-test + type: event + status: Draft tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - count: 2 - example-3: - summary: Dataset list success when type filter provided - value: + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample-trip-details + name: sample-trip-details + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 30 + example-1: + summary: 'Success: Filter based on status as array' + value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:57:28+05:30' + ts: '2024-07-17T17:57:38+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 31aba5bc-8492-45ce-be0e-8c52d8716014 + responseCode: OK + result: + data: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + count: 16 + example-2: + summary: 'Success: Filter based on status as string' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T17:59:18+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a08c7ea0-bb1c-4998-b47d-a76e38e87e31 + responseCode: OK + result: + data: + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 8 + example-3: + summary: 'Success: Filter based on dataset type' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:00:41+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 0d1ff2de-42c9-4192-b75d-84f711dbfb55 + responseCode: OK + result: + data: + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + count: 2 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '312' + ETag: + schema: + type: string + example: W/"138-XQplwhrgIYKIg0qtQdRCYWIGTNM" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:32:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:02:26+05:30' + params: + status: FAILED + resmsgid: add9dbe0-f362-4f99-890c-3387c998a049 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_LIST_INPUT_INVALID + message: >- + #properties/params/required must have required property + 'msgid' + /v2/datasets/dataschema: + post: + tags: + - Dataset API's + summary: Schema Generation + description: This api is used to generate data schema for the given dataset event. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + data: + - eid: IMPRESSION + ets: 1672657002221 + ver: '3.0' + mid: IMPRESSION:2b5834e196f485c17c4e49d292af43c0 + actor: + id: 0c45959486f579c24854d40a225d6161 + type: User + context: + channel: '01268904781886259221' + pdata: + id: staging.diksha.portal + ver: 5.1.0 + pid: sunbird-portal + env: public + sid: 23850c90-8a8c-11ed-95d0-276800e1048c + did: 0c45959486f579c24854d40a225d6161 + cdata: [] + rollup: + l1: '01268904781886259221' + uid: anonymous + object: {} + tags: + - '01268904781886259221' + edata: + type: view + pageid: login + subtype: pageexit + uri: >- + https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4 + visits: [] + syncts: 1672657005814 + '@timestamp': '2023-01-02T10:56:45.814Z' + flags: + ex_processed: true + - eid: IMPRESSION + ets: 1672656997928 + ver: '3.0' + mid: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: POST + - url: /v1/org/search + - duration: 0 + - status: OK + - eid: LOG + ets: 1672656998024 + ver: '3.0' + mid: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + type=system, message=EXIT LOG: method : POST, url: + /v1/org/search , For Operation : orgSearch, + params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + errmsg=Invalid value null for parameter hashTagId. + Please provide a valid value., + resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + err=UOS_ORGSER0017, status=FAILED, + responseCode=400}]}} + params: [] + - eid: LOG + ets: 1672657004961 + ver: '3.0' + mid: f34112c7242a3e3a26f0015796b029c2 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: f34112c7242a3e3a26f0015796b029c2 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + ElasticSearchRestHighImpl:search: calling search for + index org_alias, with query = + {"from":0,"size":250,"query":{"bool":{"must":[{"term":{"isTenant.raw":{"value":true,"boost":1.0}}},{"term":{"slug.raw":{"value":"ntp","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":[],"excludes":[]}} + params: [] + - eid: LOG + ets: 1672657006595 + ver: '3.0' + mid: d23ff123-40f0-4262-a69b-b75b46d315a1 + actor: + id: 930a3994-cbe7-4e84-936f-4974096af6f2 + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: d23ff123-40f0-4262-a69b-b75b46d315a1 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, + type=system, message=ENTRY LOG: method : GET, url: + /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 + , For Operation : getUserRolesById, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006611 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, + type=system, message=ENTRY LOG: method : GET, url: + /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , + For Operation : getUserProfileV5, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006620 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user_roles + WHERE userId=?; + params: [] + - eid: LOG + ets: 1672657006645 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: GET + - url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 + - duration: 0 + - status: OK + - eid: LOG + ets: 1672657007238 + ver: '3.0' + mid: d4d34fde-c407-efb6-03bd-9f892ca0f114 + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.portal + pid: learner-service + ver: 5.0.0 + env: User + did: d904c90d9f81ddac20141b94ddd606a0 + cdata: + - id: d4d34fde-c407-efb6-03bd-9f892ca0f114 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user WHERE + id=?; + params: [] + config: + dataset: financial_transactions + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '10760' + ETag: + schema: + type: string + example: W/"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:02:50 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:32:50+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1309aea0-9a97-46e9-bc5e-a16a8a7fb624 + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' format + type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'sid'. The property sid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.sid + - message: >- + The Property 'context.sid' appears to be + 'uuid' format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.context.properties.sid + arrival_format: text + data_type: string + did: + type: string + arrival_format: text + data_type: string + cdata: + type: array + items: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'context.cdata[*].id' + appears to be 'uuid' format type. + advice: >- + Suggest to not to index the high + cardinal columns + resolutionType: DEDUP + severity: LOW + path: >- + properties.context.properties.cdata.items.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: array + data_type: array + rollup: + type: object + properties: + l1: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at + property: 'l1'. The property l1: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System + has updated the property schema to + optional + resolutionType: OPTIONAL + severity: MEDIUM + path: >- + properties.context.properties.rollup.properties.l1 + arrival_format: text + data_type: string + arrival_format: object + data_type: object + uid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uid'. The property uid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.uid + arrival_format: text + data_type: string + arrival_format: object + data_type: object + object: + type: object + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'object'. The property object: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.object + arrival_format: object + data_type: object + tags: + type: array + items: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'tags'. The property tags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.tags + arrival_format: array + data_type: array + edata: + type: object + properties: + type: + type: string + arrival_format: text + data_type: string + pageid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'pageid'. The property pageid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.pageid + arrival_format: text + data_type: string + subtype: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'subtype'. The property subtype: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.subtype + arrival_format: text + data_type: string + uri: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uri'. The property uri: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.uri + - message: >- + The Property 'edata.uri' appears to be 'uri' + format type. + severity: '' + path: properties.edata.properties.uri + arrival_format: text + data_type: string + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'visits'. The property visits: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.visits + arrival_format: array + data_type: array + level: + type: string + arrival_format: text + data_type: string + message: + type: string + arrival_format: text + data_type: string + params: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + syncts: + type: integer + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'syncts'. The property syncts: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.syncts + - message: >- + The Property 'syncts' appears to be 'epoch' format + type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + '@timestamp': + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + '@timestamp'. The property @timestamp: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.@timestamp + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + flags: + type: object + properties: + ex_processed: + type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'ex_processed'. The property ex_processed: + only 1 time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags.properties.ex_processed + arrival_format: boolean + data_type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'flags'. The property flags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + mid: + path: $.mid + cardinality: 67 + index: false + actor.id: + path: $.actor.properties.id + cardinality: 56 + index: false + context.sid: + path: $.context.properties.sid + cardinality: 11 + index: true + edata.uri: + path: $.edata.properties.uri + cardinality: 11 + index: true + context.cdata[*].id: + path: $.context.properties.cdata.items.properties.id + cardinality: 62 + index: false + processing: + dedupKeys: + - mid + - context.cdata[*].id + - actor.id + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:03:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request body' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:33:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bbcc86c2-042d-4f77-bb6e-e1c9116df570 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'data' + example-1: + summary: 'Failure: Invalid request (config not provided)' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:35:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1f856c5e-37f0-41e9-96fb-642471228da2 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'config' + /v2/datasets/status-transition: + post: + tags: + - Dataset API's + summary: Dataset Status Transition + description: >- + This API allows you to perform status transition between 2 states. + Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to + Live, Live to Retired and even Delete a dataset. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: ReadyToPublish + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete success: Deleted dataset successfully' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T12:18:54+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + responseCode: OK + result: + message: Dataset status transition to Delete successful + dataset_id: trip-data + example-1: + summary: 'Live success: Dataset published successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:21:42+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + responseCode: OK + result: + message: Dataset status transition to Live successful + dataset_id: telemetry_record + example-2: + summary: 'ReadyToPublish success: Dataset is ready to publish' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:30:04+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + responseCode: OK + result: + message: Dataset status transition to ReadyToPublish successful + dataset_id: telemetry-events + example-3: + summary: 'Retire success: Dataset retired successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:22:58+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + responseCode: OK + result: + message: Dataset status transition to Retire successful + dataset_id: sb-telemetry + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Live failure: Dataset in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T15:37:43+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + responseCode: BAD_REQUEST + error: + code: DATASET_LIVE_FAILURE + message: >- + Transition failed for dataset: sb-telemetry2 + status:Draft with status transition to Live + trace: '' + example-1: + summary: 'ReadyToPublish failure: Incomplete dataset configs' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:36:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + responseCode: BAD_REQUEST + error: + code: DATASET_CONFIGS_INVALID + message: >- + #properties/denorm_config/properties/denorm_fields/items/required + must have required property 'dataset_name' + trace: '' + example-2: + summary: 'ReadyToPublish failure: Dataset not in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:38:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + responseCode: BAD_REQUEST + error: + code: DATASET_READYTOPUBLISH_FAILURE + message: >- + Transition failed for dataset: telemetry-events + status:Retired with status transition to ReadyToPublish + trace: '' + example-3: + summary: 'Retire Failure: Dataset is already retired' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:42:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 + responseCode: BAD_REQUEST + error: + code: DATASET_RETIRE_FAILURE + message: >- + Transition failed for dataset: master-telemetrry + status:Retired with status transition to Retire + trace: '' + example-4: + summary: >- + Retire failure: Cannot retire master dataset as it is used + by other datasets + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:01:41+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 + responseCode: BAD_REQUEST + error: + code: DATASET_IN_USE + message: >- + Failed to retire dataset as it is in use. Please retire + or delete dependent datasets before retiring this + dataset + trace: '' + example-5: + summary: 'Failure: Invalid request payload provided' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:03:56+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 + responseCode: BAD_REQUEST + error: + code: DATASET_STATUS_INVALID_INPUT + message: >- + #properties/request/properties/status/enum should be + equal to one of the allowed values + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete failure: Dataset not found to delete' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:25:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: master' + trace: '' + example-1: + summary: 'Live failure: Dataset not found to publish' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:35:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: telemetry-data' + trace: '' + example-2: + summary: 'Retire Failure: Dataset not found to retire' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:40:31+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: sb-telemetry2' + trace: '' + /v2/datasets/import: + post: + tags: + - Dataset API's + summary: Dataset import + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + id: sb-telemetry_draft_data + dataset_id: sb-telemetryRPF_draft_data + name: sb-telemetry_draft_data + type: event + extraction_config: + is_batch_event: false + dedup_config: + drop_duplicates: false + dedup_key: id + dedup_period: 604800 + extraction_key: events + validation_config: + validate: true + mode: Strict + dedup_config: + dedup_key: id + drop_duplicates: true + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + syncts: + type: integer + arrival_format: number + data_type: integer + ets: + type: integer + arrival_format: number + data_type: epoch + suggestions: + - message: >- + The Property 'ets' appears to be 'date-time' + format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.ets + flags: + type: object + properties: + ex_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_validation_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_duplicate_skipped: + type: boolean + arrival_format: boolean + data_type: boolean + user_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + device_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + loc_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + content_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + coll_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + arrival_format: object + data_type: object + additionalProperties: false + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + edata: + type: object + properties: + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + type: + type: string + arrival_format: text + data_type: string + duration: + type: number + arrival_format: number + data_type: number + size: + type: integer + arrival_format: number + data_type: integer + query: + type: string + arrival_format: text + data_type: string + filters: + type: object + properties: + objectType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + version: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + status: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + id: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + isRootOrg: + type: boolean + arrival_format: boolean + data_type: boolean + trackable.enabled: + type: string + arrival_format: text + data_type: string + channel: + type: object + arrival_format: object + data_type: object + additionalProperties: false + framework: + type: object + arrival_format: object + data_type: object + additionalProperties: false + resourceType: + type: object + arrival_format: object + data_type: object + additionalProperties: false + identifier: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + contentType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + mimeType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + hashTagId: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + compatibilityLevel: + type: object + properties: + min: + type: integer + arrival_format: number + data_type: integer + max: + type: integer + arrival_format: number + data_type: integer + arrival_format: object + data_type: object + additionalProperties: false + createdBy: + type: string + arrival_format: text + data_type: string + mediaType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + origin: + type: string + arrival_format: text + data_type: string + primaryCategory: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + trackable: + enabled: + type: string + arrival_format: object + data_type: object + additionalProperties: false + sort: + type: object + properties: + lastUpdatedOn: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + topn: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + pageid: + type: string + arrival_format: text + data_type: string + uri: + type: string + arrival_format: text + data_type: string + subtype: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + data: + type: string + arrival_format: text + data_type: string + uaspec: + type: object + properties: + agent: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + system: + type: string + arrival_format: text + data_type: string + platform: + type: string + arrival_format: text + data_type: string + raw: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + state: + type: string + arrival_format: text + data_type: string + props: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + prevstate: + type: string + arrival_format: text + data_type: string + dspec: + type: object + properties: + os: + type: string + arrival_format: text + data_type: string + make: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + idisk: + type: number + arrival_format: number + data_type: number + edisk: + type: number + arrival_format: number + data_type: number + scrn: + type: number + arrival_format: number + data_type: number + camera: + type: string + arrival_format: text + data_type: string + cpu: + type: string + arrival_format: text + data_type: string + sims: + type: integer + arrival_format: number + data_type: integer + webview: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + extra: + type: object + properties: + pos: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + values: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + query: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + mode: + type: string + arrival_format: text + data_type: string + duration: + type: string + arrival_format: object + data_type: object + additionalProperties: false + '@timestamp': + type: string + suggestions: + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + l2: + type: string + arrival_format: text + data_type: string + l3: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + did: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + object: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + version: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + tags: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + additionalProperties: false + denorm_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + denorm_fields: + - denorm_key: actor.id + dataset_id: master-dataset + denorm_out_field: userdata + router_config: + topic: sb-telemetry + dataset_config: + keys_config: + timestamp_key: obsrv_meta.syncts + data_key: '' + partition_key: '' + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + cache_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + redis_db: 0 + file_upload_path: [] + transformations_config: [] + connectors_config: [] + tags: + - tag1 + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1724333643940' + api_version: v2 + sample_data: {} + entry_topic: dev.ingest + created_date: '2024-07-23T18:35:15.690Z' + updated_date: '2024-08-22T13:34:08.041Z' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: overwrite + in: query + schema: + type: boolean + example: 'false' + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '346' + ETag: + schema: + type: string + example: W/"15a-SX5o6plRdfy+akEYhbMwq9zl+oU" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:34:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Fully imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:04:26+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: fd87393e-4b1b-4cf0-8e9a-a0529d691cc3 + responseCode: OK + result: + message: Dataset is imported successfully + data: + id: sb-telemetry_draft_data + version_key: '1732001666671' + example-1: + summary: 'Success: Partially imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T12:59:41+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 7d9aeb08-2952-485f-9944-4d118da7842e + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 379ced0b-eb06-445f-a7e3-8737cb16c351 + responseCode: OK + result: + message: Dataset is partially imported + data: + id: sb-telemetry_draft_data + version_key: '1732001381825' + ignoredFields: + denorm_fields: + - config: + denorm_key: actor.id + denorm_out_field: userdata + dataset_id: master-dataset + details: Master dataset does not exist + example-2: + summary: 'Success: Imported new dataset [overwrite=false]' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:05:32+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1a50e1b1-14b0-4178-aba4-9e09a63d8d1d responseCode: OK result: + message: Dataset is imported successfully data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema + id: sb-telemetry_draft + version_key: '1732001732045' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-Dxem7prJM89DOqth+bdZbh0UI1g" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:38:05 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:08:05+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 244318a5-d933-41a5-add6-14f299281069 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_IMPORT_INVALID_CONFIGS + message: >- + #properties/request/required must have required property + 'id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '351' + ETag: + schema: + type: string + example: W/"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:31:11 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:01:11+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1e5b34cd-4473-4439-8cc5-5ff917f47e48 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: >- + Dataset with dataset_id: sb-telemetryRPF_draft_data already + exists. + /v2/datasets/export/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset + get: + tags: + - Dataset API's + summary: Export Dataset + parameters: + - name: status + in: query + schema: + type: string + example: Live + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '5155' + ETag: + schema: + type: string + example: W/"1423-Tx46QCrIX8So3vSGVkd865IYeEY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:16:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Live Dataset export success' + value: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:46:01+05:30' + params: + status: SUCCESS + resmsgid: ae57c620-2858-40f9-9d8e-33f8f259449d + responseCode: OK + result: + id: content-data-v2 + dataset_id: content-data-v2 + type: master + name: content-data-v2 + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true properties: - skill: + email: + key: email type: string arrival_format: text data_type: string - name: + isRequired: false + resolved: true + mobile: + key: mobile type: string arrival_format: text data_type: string - skill_id: + isRequired: false + resolved: true + fare_details: + key: fare_details + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + fare_amount: + key: fare_amount type: string arrival_format: text data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry - dataset_config: - data_key: skill - timestamp_key: '' - exclude_fields: [] - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 4 - tags: [] - status: Publish - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 1 - example-4: - summary: Dataset list success based on sortBy values + isRequired: false + resolved: true + extra: + key: extra + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mta_tax: + key: mta_tax + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tolls_amount: + key: tolls_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + improvement_surcharge: + key: improvement_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + total_amount: + key: total_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + congestion_surcharge: + key: congestion_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + additionalProperties: true + dedup_config: + drop_duplicates: true + dedup_key: tripID + dedup_period: 604800 + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: content-data-v2 + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-2_addedf.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + timestamp_key: tpep_pickup_datetime + data_key: '' + partition_key: '' + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 7 + tags: [] + status: Live + created_by: SYSTEM + updated_by: SYSTEM + data_version: 1 + api_version: v2 + version: 1 + sample_data: + mergedEvent: + tripID: ccbb970c-4d10-4247-be0c-a480996f3429 + VendorID: '1' + tpep_pickup_datetime: '2023-02-26 00:36:10' + tpep_dropoff_datetime: '2023-09-15 00:45:45' + passenger_count: '2' + trip_distance: '1.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '263' + payment_type: '1' + primary_passenger: + email: Jacey_Hintz@yahoo.com + mobile: 247-492-3370 + fare_details: + fare_amount: '8.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.45' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '12.25' + congestion_surcharge: '' + entry_topic: local.ingest + created_date: '2024-10-14T06:23:44.588Z' + updated_date: '2024-10-14T06:23:44.588Z' + example-1: + summary: 'Success: ReadyToPublish Dataset exported successfully' value: - id: api.datasets.list + id: api.datasets.export ver: v2 - ts: '2024-04-19T12:58:27+05:30' + ts: '2024-11-19T12:48:11+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 270a4e50-f7e5-4562-ae75-7a967866d065 + resmsgid: 222847a0-32bf-4c63-be68-76f0e51258af responseCode: OK result: - data: - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema + id: mydataset + dataset_id: mydataset + name: mydataset + type: event + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + validation_config: + validate: true + mode: Strict + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true properties: - skill: - type: string - arrival_format: text - data_type: string - name: + email: + key: email type: string arrival_format: text data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: + isRequired: false + resolved: true + mobile: + key: mobile type: string arrival_format: text data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry - dataset_config: - data_key: skill - timestamp_key: '' - exclude_fields: [] - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 4 - tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' - version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema + isRequired: false + resolved: true + additionalProperties: false + fare_details: + key: fare_details type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry - dataset_config: - data_key: skill - timestamp_key: '' - exclude_fields: [] - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 4 - tags: [] - status: Publish - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 2 + fare_amount: + key: fare_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + extra: + key: extra + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + mta_tax: + key: mta_tax + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + tolls_amount: + key: tolls_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + improvement_surcharge: + key: improvement_surcharge + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + total_amount: + key: total_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + congestion_surcharge: + key: congestion_surcharge + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + additionalProperties: false + additionalProperties: false + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: mydataset + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-1 (4) + (1)_5ec855.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 + transformations_config: [] + connectors_config: [] + tags: [] + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1731420131072' + api_version: v2 + sample_data: + mergedEvent: + tripID: 4c77e9d5-538d-4eb7-8db1-4c2c32860aa8 + VendorID: '2' + tpep_pickup_datetime: '2023-11-02 00:18:42' + tpep_dropoff_datetime: '2023-11-02 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Willa67@gmail.com + mobile: 1-720-981-6399 x77055 + fare_details: + fare_amount: 7 + extra: 0.5 + mta_tax: 0.5 + tip_amount: 0 + tolls_amount: 0 + improvement_surcharge: 0.3 + total_amount: 8.3 + congestion_surcharge: 0 + entry_topic: local.ingest + created_date: '2024-11-11T21:31:21.390Z' + updated_date: '2024-11-12T08:32:27.931Z' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '291' + ETag: + schema: + type: string + example: W/"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:20:43 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object example: - id: api.datasets.list + id: api.datasets.export ver: v2 - ts: '2024-04-19T12:58:47+05:30' + ts: '2024-11-19T12:50:43+05:30' params: status: FAILED - resmsgid: f9e1fae9-4660-409d-bb4d-e29888983d4b + resmsgid: 2097228c-7872-4217-a06a-0ec08e3dc67d responseCode: BAD_REQUEST + result: {} error: - code: DATASET_LIST_INPUT_INVALID - message: >- - #properties/params/additionalProperties should NOT have - additional properties - trace: '' - '500': - description: Internal Server Error + code: DATASET_EXPORT_FAILURE + message: Dataset with status:Draft cannot be exported + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:15:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object example: - id: api.datasets.list + id: api.datasets.export ver: v2 - ts: '2024-04-19T12:59:15+05:30' + ts: '2024-11-19T12:45:10+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 81c3a4a5-e8ac-41de-be00-6d0300ecb106 - responseCode: INTERNAL_SERVER_ERROR + resmsgid: fd6fdae2-a002-4c15-81e2-3cce539a2a5a + responseCode: NOT_FOUND + result: {} error: - code: DATASET_LIST_FAILURE - message: Failed to list dataset - trace: '' - /v2/datasets/diff/{dataset_id}: - get: + code: DATASET_NOT_FOUND + message: >- + Dataset with the given dataset_id:v1-copy not found to + export + /v2/datasets/copy: + post: tags: - - Dataset CRUD APIs - summary: Dataset Diff - description: >- - This API allows you to return the difference between the live and draft - dataset by returning additions, deletions and modifications in the - dataset. + - Dataset API's + summary: Clone Dataset + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + source: + datasetId: dataset-telemetry + isLive: true + destination: + datasetId: bew-copy-live2 parameters: - - name: dataset_id - example: telemetry - in: path - required: true + - name: Content-Type + in: header schema: type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '296' + ETag: + schema: + type: string + example: W/"128-bi4wSfgkyMP13qFE8VPTmgDdgLA" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:26:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:56:12+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 28072631-8e80-45eb-906f-d933a90646d0 + responseCode: OK + result: + dataset_id: new-copy-live2 + message: Dataset clone successful + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '328' + ETag: + schema: + type: string + example: W/"148-jUDshghYf/jOhmmVkvtHlfJT864" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:22:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:52:41+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: f2fd51bb-fc18-4278-a229-47fa25398e69 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_EXISTS + message: Dataset dataset-telemetry does not exists + /v2/connectors/list: + post: + tags: + - Connector API's + summary: List Connectors + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Draft responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '250' + ETag: + schema: + type: string + example: W/"fa-+eWKIfUxsWBGuJy23qSucgLXke4" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:55:51 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Data Format Diff + summary: 'Success: Filtered based on status' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T16:55:25+05:30' + ts: '2024-07-30T15:25:51+05:30' params: status: SUCCESS - resmsgid: c58aa5f7-f182-4dff-9ee9-16f508e53c51 + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a responseCode: OK result: - additions: - - type: dataFormat - items: - - name: extraction_key - value: mid - - name: batch_id - value: id - - name: dedup_config.dedup_key - value: id - modifications: - - type: dataFormat - items: - - name: is_batch_event - value: - from: false - to: true - deletions: [] + data: [] + count: 0 example-1: - summary: Timestamp and data key diff + summary: 'Success: Filtered based on category' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T17:00:44+05:30' + ts: '2024-07-31T18:55:03+05:30' params: status: SUCCESS - resmsgid: 8ec5fa57-5105-4793-ae96-3a7e0550dbe9 + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab responseCode: OK result: - additions: [] - modifications: - - type: timestamp - value: - from: syncts - to: ets - - type: dataKey - value: - from: eid - to: mid - deletions: [] + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + count: 4 example-2: - summary: Data Schema Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-28T15:40:48+05:30' - params: - status: SUCCESS - resmsgid: c364416d-b903-4af6-a39b-a4e52c4bf6b5 - responseCode: OK - result: - additions: - - type: dataSchema - items: - - name: eid_1 - value: - arrivalFormat: boolean - dataType: boolean - absolutePath: $.properties.eid_1 - isRequired: false - modifications: - - type: dataSchema - items: - - field: eid - name: type - value: - from: string - to: boolean - - field: eid - name: arrivalFormat - value: - from: text - to: boolean - - field: eid - name: dataType - value: - from: string - to: boolean - deletions: - - type: dataSchema - items: - - name: ver - example-3: - summary: Data Validation Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:05:28+05:30' - params: - status: SUCCESS - resmsgid: ad6b4977-c341-4193-a42f-9b45993e5f8e - responseCode: OK - result: - additions: [] - modifications: - - type: validation - items: - - field: mode - value: - from: IgnoreNewFields - to: Strict - deletions: [] - example-4: - summary: Dataset Dedup diff + summary: 'Success: Connectors list with all filter options' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T17:07:58+05:30' + ts: '2024-07-31T18:56:32+05:30' params: status: SUCCESS - resmsgid: e59b841b-e35f-462d-adb1-31018d718f6a - responseCode: OK - result: - additions: - - type: dedup - items: - - name: dedup_key - value: mid - modifications: - - type: dedup - items: - - name: drop_duplicates - value: - from: false - to: true - deletions: [] - example-5: - summary: Dataset Denorm Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:17:35+05:30' - params: - status: SUCCESS - resmsgid: ee558c17-b3a6-41cc-9b07-64105053394e + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 11a2f537-bd98-405b-97e5-0f0d5b86b2c3 responseCode: OK result: - additions: - - type: denorm - items: - - name: syncts - value: - denorm_key: actor.id - dataset_name: mid - modifications: [] - deletions: - - type: denorm - items: - - name: mid - value: - denorm_key: actor.id - dataset_name: mid - example-6: - summary: Dataset transformation diff + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-3: + summary: 'Success: Connectors list without filters' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T17:25:12+05:30' + ts: '2024-07-31T18:57:37+05:30' params: status: SUCCESS - resmsgid: f4389891-9c09-4ccd-affe-39970590045b + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c2467e01-0a2d-401c-aa3d-dd16b804f723 responseCode: OK result: - additions: - - type: transformations - items: - - name: skill_id - value: - transformation_function: - type: mask - condition: null - expr: skill_id - mode: Strict - metadata: - section: pii - modifications: - - type: transformations - items: - - name: new - field: transformation_function.type - value: - from: jsonata - to: encrypt - metadata: - section: transformation - deletions: - - type: transformations - items: - - name: mid - value: - metadata: - section: transformation - - name: name - value: - metadata: - section: transformation - - name: neew - value: - metadata: - section: additionalFields - /v2/files/generate-url: - post: - tags: - - Dataset CRUD APIs - summary: Files generate URL - description: >- - This API allows you to generate pre signed urls from cloud providers for - getting and putting files. On access: write, upload urls are generated to put files and on access: read, download urls are generated to get files. - requestBody: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '399' + ETag: + schema: + type: string + example: W/"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:43:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: - examples: - example1: - summary: Generate upload URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: write - example2: - summary: Generate download URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: read + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:13:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fadde0-8c59-4420-8ab3-56474b01670b + responseCode: BAD_REQUEST + result: {} + error: + code: CONNECTORS_LIST_INPUT_INVALID + message: >- + #properties/request/properties/filters/properties/status/minItems + must NOT have fewer than 1 items + /v2/connectors/read/{connector_id}: + parameters: + - name: connector_id + in: path + required: true + schema: + type: string + description: Unique identifier for the connector + get: + tags: + - Connector API's + summary: Read Connector responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '3304' + ETag: + schema: + type: string + example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 12:47:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Success: Generate Put URL' + summary: 'Success: Read live Connector' value: - id: api.files.generate-url + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:04:15+05:30' + ts: '2024-07-31T18:17:54+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a79f1260-1b34-4b96-9c12-cf0f5600e9f3 + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced responseCode: OK result: - - filePath: >- - obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=0zWi7R9wqUT6uW0QuN%2F0nyvaloSCqRPTvzHn%2BpkCWdk%3D - - filePath: >- - obsrv-onest/api-service/user_uploads/school_data_14092a.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data_14092a.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=35g%2F46r%2FOI490%2B8pKry9mEbAgXOXznmCyw7L%2FSLITM0%3D + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: + type: object + properties: + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: + type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly + fieldDescription: + - type: string + description: '' + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: + batch_size: + ui:readonly: true + max_batches: + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' example-1: - summary: 'Success: Generate Get URL' + summary: 'Success: Read Draft Connector' value: - id: api.files.generate-url + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:05:47+05:30' + ts: '2024-08-01T12:47:12+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 141717fb-9b94-463d-b0f1-1bab40b187cc + resmsgid: b6fcfb05-246c-4a1b-9eb1-27497ee9b80b responseCode: OK result: - - filePath: obsrv-onest/api-service/user_uploads/telemetry.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=UEWKMP9wcETNKA1sXo%2BVI6qZ7Iqk%2BJg6kglaEXz7%2B1Y%3D - - filePath: obsrv-onest/api-service/user_uploads/school_data.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=jXzootTAoUAWLYKF9AYwKh9Mzl%2FVctlebTBHVz%2BbaKM%3D - '400': - description: Bad Request + id: mssql-connector-2.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 2.0.0 + description: >- + The MS SQL Connector is used to move data from any MS + SQL Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Draft + ui_spec: {} + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '276' + ETag: + schema: + type: string + example: W/"114-izVC8DsHdeSfau/USVJvnIqZIMQ" + Date: + schema: + type: string + example: Thu, 01 Aug 2024 09:32:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - examples: - example-0: - summary: 'Failure: Invalid access value requested' - value: - id: api.files.generate-url - ver: v2 - ts: '2024-05-30T12:08:23+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b1da48a0-5381-49a1-9d5f-ce6c7c65fdcf - responseCode: BAD_REQUEST - error: - code: FILES_GENERATE_URL_INPUT_INVALID - message: >- - #properties/request/properties/access/enum should be - equal to one of the allowed values - trace: '' - example-1: - summary: >- - Failure: Number of file url requested exceeds the limit - configured - value: - id: api.files.generate-url - ver: v2 - ts: '2024-05-30T12:10:29+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5828abd2-2934-40ef-8fa1-a9bca1e7e24e - responseCode: BAD_REQUEST - error: - code: FILES_URL_GENERATION_LIMIT_EXCEED - message: 'Pre-signed URL generation failed: limit exceeded.' - trace: '' - /v2/obsrv/data/sql-query: + example: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T15:02:48+05:30' + params: + status: FAILED + resmsgid: 712e7298-99f8-4694-9011-4232fcfd664a + responseCode: NOT_FOUND + result: {} + error: + code: CONNECTOR_NOT_FOUND + message: 'Connector not found: postgres-conn' + /v2/connector/register: post: tags: - - Dataset CRUD APIs - summary: SQL Wrapper - description: This api is a wrapper api, used to query the data. + - Connector API's + summary: Register Connector + description: Registers a connector by uploading a file. + operationId: registerConnector requestBody: + required: true content: - application/json: + multipart/form-data: schema: type: object - example: - query: >- - SELECT "channel","flags", COUNT(*) AS "Count" FROM "wikipedia" - WHERE "page" IS NOT NULL GROUP BY 1, 2 ORDER BY 3 DESC + properties: + file3: + type: string + format: binary + description: Connector distribution file to upload. responses: '200': - description: OK - content: - application/json: - schema: - type: object - example: - - channel: '#en.wikipedia' - flags: '' - Count: 4776 - - channel: '#fr.wikipedia' - flags: '!M' - Count: 31 - - channel: '#fa.wikipedia' - flags: B - Count: 30 - - channel: '#vi.wikipedia' - flags: 'N' - Count: 9 - - channel: '#ar.wikipedia' - flags: '!N' - Count: 8 - - channel: '#da.wikipedia' - flags: '!' - Count: 8 - - channel: '#hy.wikipedia' - flags: '!N' - Count: 8 - - channel: '#id.wikipedia' - flags: '!N' - Count: 8 - - channel: '#eu.wikipedia' - flags: M - Count: 3 - - channel: '#fa.wikipedia' - flags: '!N' - Count: 3 - - channel: '#fi.wikipedia' - flags: 'N' - Count: 3 - - channel: '#he.wikipedia' - flags: NB - Count: 3 - - channel: '#hi.wikipedia' - flags: '' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!N' - Count: 1 - '400': - description: Bad Request + description: Connector registered successfully. content: application/json: schema: type: object - example: - id: api.obsrv.data.sql-query - ver: v2 - ts: '2024-05-30T16:31:57+05:30' - params: - status: FAILED - resmsgid: 7709ef5b-d7f3-48a5-bf6c-cf243fb6d2eb - responseCode: BAD_REQUEST - error: - code: SQL_QUERY_EMPTY_REQUEST - message: Failed to query. Invalid request - trace: '' + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:28:47+00:00" + params: + type: object + properties: + status: + type: string + example: SUCCESS + resmsgid: + type: string + example: fc2ce24f-a333-479f-931c-024d61039801 + responseCode: + type: string + example: OK + result: + type: object + properties: + message: + type: string + example: connector registered successfully. '500': - description: Internal Server Error + description: Internal server error while registering the connector. content: application/json: schema: type: object - example: - id: api.obsrv.data.sql-query - ver: v2 - ts: '2024-05-29T13:03:44+05:30' - params: - status: FAILED - resmsgid: 0b4b7168-2b70-406b-84ce-c320b65559eb - responseCode: INTERNAL_SERVER_ERROR - error: - code: ERR_BAD_RESPONSE - message: Request failed with status code 500 - trace: '' - /v2/datasets/status-transition: + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:50:21+00:00" + params: + type: object + properties: + status: + type: string + example: FAILED + resmsgid: + type: string + example: ebaad9d7-c0d4-47e4-8cde-87fb25eef739 + responseCode: + type: string + example: INTERNAL_SERVER_ERROR + error: + type: object + properties: + code: + type: string + example: FAILED_TO_REGISTER_CONNECTOR + message: + type: string + example: Failed to Extract the File + trace: + type: string + example: "" + /v2/data/in/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset post: tags: - - Dataset CRUD APIs - summary: Dataset Status Transition - description: >- - This API allows you to perform status transition between 2 states. - Allowed status transition are: -
- - Draft to ReadyToPublish when the dataset configuration is complete and configuration is validated, ReadyToPublish to Live to make dataset live and make it actively running within the pipeline, Live to Retired when dataset is needed to be stopped and retired.User can even Delete the dataset of Draft and ReadyToPublish status. + - Data Ingest + summary: Ingest Data requestBody: - content: - application/json: - examples: - example1: - summary: Transition to ReadyToPublish - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: ReadyToPublish - example2: - summary: Transition to Live - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Live - example3: - summary: Transition to Retired - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Retire - example4: - summary: Delete a draft dataset - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Delete + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + data: + eid: INTERACT + date: '2022-01-01' + ver: '3.0' + syncts: 1668591949682 + ets: 1668591949682 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '261' + ETag: + schema: + type: string + example: W/"105-UtaCmh0qZMBRBdniNq74Gr+4YAo" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:05:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete success: Deleted dataset successfully' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T12:18:54+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b - responseCode: OK - result: - message: Dataset deleted successfully - dataset_id: master.1 - example-1: - summary: 'Live success: Dataset published successfully' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T12:21:42+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 - responseCode: OK - result: - message: Dataset published successfully - dataset_id: telemetry - example-2: - summary: 'ReadyToPublish success: Dataset is ready to publish' + summary: 'Success: Data ingested successfully' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:30:04+05:30' + ts: '2024-05-30T23:35:48+05:30' params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 - responseCode: OK - result: - message: Dataset status transition to ReadyToPublish successful - dataset_id: telemetry-events - example-3: - summary: 'Retire success: Dataset retired successfully' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 9a44ac5b-ef82-46f7-92c5-c5b39ef88764 + responseCode: OK + result: + message: Data ingested successfully + example-1: + summary: 'Success: Data ingested successfully (batch)' value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T12:22:58+05:30' + ts: '2024-05-30T23:36:49+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: a220041c-0c28-415a-9687-9fb2e211f8c4 responseCode: OK result: - message: Dataset retired successfully - dataset_id: sb-telemetry + message: Data ingested successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '300' + ETag: + schema: + type: string + example: W/"12c-GNCIUUDxOZD3UfM311sjnmFIgPc" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:03:32 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '2024-05-30T23:33:32+05:30' + params: + status: FAILED + resmsgid: acf07609-77de-4ab5-81ea-42e41b8b95ff + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_INGESTION_INVALID_INPUT + message: '#required should have required property ''id''' + trace: '' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '316' + ETag: + schema: + type: string + example: W/"13c-O9iirC/EyneYXQzth7iwEEy1UV4" + Date: + schema: + type: string + example: Thu, 18 Apr 2024 10:17:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Live failure: Dataset in draft state' + summary: Entry topic not found value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T15:37:43+05:30' + ts: '2024-04-18T15:47:00+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 11c32a3a-fdeb-4e00-a9cf-f6433ade5608 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_NOT_READY_FOR_PUBLISH - message: Failed to publish dataset as it is in draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' example-1: - summary: 'ReadyToPublish failure: Incomplete dataset configs' + summary: 'Failure: Dataset not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:36:16+05:30' + ts: '2024-05-30T23:37:29+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: d4db36b4-37b0-4170-a4cf-9d2ae8fa0416 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_CONFIGS_INVALID - message: >- - #properties/denorm_config/properties/denorm_fields/items/required - must have required property 'dataset_name' + code: DATASET_NOT_FOUND + message: Dataset with id not found trace: '' example-2: - summary: 'ReadyToPublish failure: Dataset not in draft state' + summary: 'Failure: Entry topic not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:38:14+05:30' + ts: '2024-05-30T23:39:26+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 62309380-3e83-442f-877e-359ed2774bbf responseCode: BAD_REQUEST + result: {} error: - code: DATASET_READYTOPUBLISH_FAILURE - message: >- - Failed to mark dataset Ready to publish as it not in - draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' - example-3: - summary: 'Retire Failure: Dataset is already retired' + /v2/data/query/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset + post: + tags: + - Data Query + summary: Query Data + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + context: + aggregationLevel: day + query: >- + Select * from "check" WHERE __time >= timestamp '2020-12-31' + AND __time < TIMESTAMP '2024-01-21' Limit 1 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '313' + ETag: + schema: + type: string + example: W/"139-De+IthAwrGNR+J11CwlNf5RSMmw" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:17:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: native query (interval as array type)' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:42:18+05:30' + ts: '2024-05-30T23:47:45+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_RETIRED - message: Dataset is already retired - trace: '' - example-4: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 134efe35-c096-4cab-ad14-db6a8952f264 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-1: + summary: 'Success: native query (queryType: scan)' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:51:14+05:30' + params: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 4fc6e3c1-4715-47b7-9137-7713fb2fbe72 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-2: summary: >- - Retire failure: Cannot retire master dataset as it is used - by other datasets + Success: native query (normal intervals & queryType : + timeseries) value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:01:41+05:30' + ts: '2024-05-30T23:51:49+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 - responseCode: BAD_REQUEST - error: - code: DATASET_IN_USE - message: Failed to retire dataset as it is used by other datasets - trace: '' - example-5: - summary: 'Failure: Invalid request payload provided' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75d7ae48-a3db-4367-b50f-34eb99ac3480 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + school_id: 0 + example-3: + summary: 'Success: sql query' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:03:56+05:30' + ts: '2024-05-30T23:42:12+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 - responseCode: BAD_REQUEST - error: - code: DATASET_STATUS_INVALID_INPUT - message: >- - #properties/request/properties/status/enum should be - equal to one of the allowed values - trace: '' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 2c981011-76da-3000-97f3-eafac939e59f + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '371' + ETag: + schema: + type: string + example: W/"173-OP8NcbSqLKFO92PIyUmMk0lNsXs" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:16:29 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:46:29+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: b6434700-dd92-4f64-9250-a22939e753b9 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_OUT_INVALID_DATE_RANGE + message: >- + Invalid date range! make sure your range cannot be more than + 30 days + trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '356' + ETag: + schema: + type: string + example: W/"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:13:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete failure: Dataset not found to delete' + summary: 'Failure: Datasource not found in live table' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T12:25:36+05:30' + ts: '2024-05-30T23:43:12+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: f922f120-2aea-49af-9a76-7312fe2eb266 responseCode: NOT_FOUND + result: {} error: - code: DATASET_NOT_FOUND - message: Dataset not found to delete + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Datasource telemetry-eventssss not available for + querying trace: '' example-1: - summary: 'Live failure: Dataset not found to publish' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T15:35:59+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c - responseCode: NOT_FOUND - error: - code: DATASET_NOT_FOUND - message: Dataset not found to publish - trace: '' - example-2: - summary: 'Retire Failure: Dataset not found to retire' + summary: 'Failure: Datasource not found in druid' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:40:31+05:30' + ts: '2024-05-30T23:50:27+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75bd4313-d504-4fd3-92ab-ee2a685beb83 responseCode: NOT_FOUND + result: {} error: - code: DATASET_NOT_FOUND - message: Dataset not found to retire - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T16:24:12+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 92928434-719f-47d4-9946-1e40ecd53253 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_STATUS_FAILURE - message: Failed to perform status transition on datasets - trace: '' - /v2/data/in/{datasetId}: - post: - tags: - - Data Ingest - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - summary: Data ingest - description: >- - This API allows you to ingest events to a data streaming platform in individual or batch format. - requestBody: - content: - application/json: - examples: - example1: - summary: Ingest individual event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: - eid: INTERACT - date: '2022-01-01' - ver: '3.0' - syncts: 1668591949682 - ets: 1668591949682 - example2: - summary: Ingest batch event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: [{ "eid": "INTERACT","date": "2022-01-01","ver": "3.0","syncts": 1668591949682}] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Data ingest successful(individual) - value: - id: api.data.in - ver: v2 - ts: 1711966306164 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - example-1: - summary: Data ingest successfully(batch) - value: - id: api.data.in - ver: v2 - ts: 1711966352987 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.data.in - ver: v2 - ts: 1711966410766 - params: - status: FAILED - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATA_INGESTION_INVALID_INPUT - message: "#required should have required property 'id'" - trace: '' - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Dataset not found - value: - id: api.data.in - ver: v2 - ts: 1711966446364 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_FOUND - message: Dataset with id not found - trace: '' - example-1: - summary: Entry topic not found - value: - id: api.data.in - ver: v2 - ts: 1711975482739 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: ENTRY_TOPIC_NOT_FOUND - message: Entry topic is not defined - trace: '' - /v2/data/query/{datasetId}: - post: - tags: - - Data query - summary: Data query - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - description: >- - This API allows you to query your datasource. - requestBody: - content: - application/json: - examples: - example1: - summary: SQL query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: >- - SELECT * FROM "rollups-configured" WHERE __time >= TIMESTAMP - '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1 - example2: - summary: Native query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - mid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: - quertType: scan - dataSource: rollups-configured - intervals: - type: intervals - intervals: ["2022-01-01/2022-02-01"] - granularity: day - columns: ["eid","__time"] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: sql query' - value: - id: api.data.out - ver: v2 - ts: 1711971972855 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - - __time: '2022-01-01T00:00:00.000Z' - eid: '1' - ver: '3.0' - flags_ex_processed: 'true' - flags_pp_validation_processed: 'true' - flags_pp_duplicate_skipped: 'true' - flags_device_denorm: 'true' - flags_user_denorm: 'true' - flags_loc_denorm: 'true' - derivedlocationdata_district: AGRA - derivedlocationdata_from: user-profile - derivedlocationdata_state: Uttar Pradesh - mid: 6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0 - type: events - actor_type: User - actor_id: 311663b2-d7de-4d46-8803-20407eaa3403 - edata_type: session - userdata_subject: null - userdata_district: AGRA - userdata_usersubtype: hm - userdata_grade: null - userdata_usersignintype: Self-Signed-In - userdata_usertype: administrator - userdata_userlogintype: administrator - userdata_state: Uttar Pradesh - devicedata_statecustomcode: '29' - devicedata_country: India - devicedata_iso3166statecode: IN-KA - devicedata_city: Bengaluru - devicedata_countrycode: IN - devicedata_state: Karnataka - devicedata_devicespec_idisk: '106.47' - devicedata_devicespec_webview: 107.0.5304.105 - devicedata_devicespec_os: Android 12 - devicedata_devicespec_scrn: '6.53' - devicedata_devicespec_sims: '-1' - devicedata_devicespec_cpu: "abi: arm64-v8a processor\t: 0 " - devicedata_devicespec_id: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - devicedata_devicespec_camera: '' - devicedata_devicespec_edisk: '106.27' - devicedata_devicespec_make: vivo 1915 - devicedata_statecode: KA - devicedata_districtcustom: BENGALURU URBAN SOUTH - devicedata_statecustomname: Karnataka - devicedata_userdeclared_district: AGRA - devicedata_userdeclared_state: Uttar Pradesh - context_cdata_id: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_cdata_type: UserSession - context_env: sdk - context_channel: '0126796199493140480' - context_pdata_id: preprod.diksha.app - context_pdata_pid: sunbird.app - context_pdata_ver: 4.10.1023preproduction - context_sid: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_did: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - context_rollup_l1: '0126796199493140480' - object_id: '' - object_type: '' - object_version: '' - object_rollup: '{}' - count: 3 - edata_duration_sum: null - example-1: - summary: 'Success: native query' - value: - id: api.data.out - ver: v2 - ts: 1711973520793 - params: - status: SUCCESS - errmsg: '' - responseCode: OK - result: - - timestamp: '2022-01-01T00:00:00.000Z' - result: - mid: 1 - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: invalid date range (sql)' - value: - id: api.data.out - ver: v2 - ts: 1711972193426 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - example-1: - summary: 'Failure: Invalid date range (native)' - value: - id: api.data.out - ver: v2 - ts: 1711972501847 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.data.out - ver: v2 - ts: 1711972085550 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.data.out - ver: v2 - ts: 1711975034302 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-2: - summary: 'Failure: Datasource not found in live table (native)' - value: - id: api.data.out - ver: v2 - ts: 1711974978635 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" - example-3: - summary: 'Failure: Datasource not found in druid(native)' - value: - id: api.data.out - ver: v2 - ts: 1711972569115 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-2: + summary: 'SQL Failure: dataset not found in druid' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:52:26+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 19849769-c290-4599-a79e-862dcbce1124 + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-3: + summary: 'SQL Failure: Datasource not found in live table' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:53:02+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 530a12f9-04f3-4986-8f52-b3184b9194bd + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource undefined not available for querying + trace: '' /v2/template/read/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template get: tags: - Query Templates - summary: Read template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + summary: Read Template responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '537' + ETag: + schema: + type: string + example: W/"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:24:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3362,10 +8088,10 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:09:55+05:30' + ts: '2024-05-30T23:54:14+05:30' params: status: SUCCESS - resmsgid: 8ef3a9d0-8658-47dd-b6cb-a4b1777a1ea4 + resmsgid: ad1c0a98-1ba3-4203-9fac-59ccaf442347 responseCode: OK result: template_id: jsontemplate111 @@ -3382,24 +8108,54 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:10:54+05:30' + ts: '2024-05-30T23:54:38+05:30' params: status: SUCCESS - resmsgid: 656185ef-527d-4b60-ac19-263da85142b8 + resmsgid: 70047866-a4b4-4fce-b9c4-4699fcab2dc6 responseCode: OK result: template_id: sql1 template_name: sql1 query: >- - "SELECT * FROM {{DATASET}} WHERE __time BETWEEN - TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP + {{ENDDATE}} lIMIT {{LIMIT}}" query_type: sql created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-04-29T11:29:58.759Z' - updated_date: '2024-04-29T11:29:58.759Z' + created_date: '2024-05-13T07:29:04.117Z' + updated_date: '2024-05-13T07:29:04.117Z' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '294' + ETag: + schema: + type: string + example: W/"126-zq5j1yor+xr1XFEEbP09fQTDI/k" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:25:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3407,30 +8163,60 @@ paths: example: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:11:45+05:30' + ts: '2024-05-30T23:55:16+05:30' params: status: FAILED - resmsgid: bc6a4621-4ac5-4959-b467-1134ea874960 + resmsgid: 958bf0e7-bdb8-4153-abdc-ab84e8004a0e responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template sql100 does not exists trace: '' /v2/template/delete/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template delete: tags: - Query Templates - summary: Delete template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + summary: Delete Template responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '241' + ETag: + schema: + type: string + example: W/"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo" + Date: + schema: + type: string + example: Tue, 30 Apr 2024 06:28:27 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3447,6 +8233,35 @@ paths: message: Template yashash-k deleted successfully '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '303' + ETag: + schema: + type: string + example: W/"12f-99pWw8VTwuVfDAhinC55JXfNyyg" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:28:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3454,27 +8269,21 @@ paths: example: id: api.query.template.delete ver: v2 - ts: '2024-04-29T18:29:09+05:30' + ts: '2024-05-30T23:58:41+05:30' params: status: FAILED - resmsgid: 6c62ff4d-0a63-4384-a365-7bbef951f288 + resmsgid: 9b7f81fb-6705-4d32-9bd3-139cd5a211b9 responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template json_template does not exists trace: '' - /v2/template/create/{template_id}: + /v2/template/create: post: tags: - Query Templates summary: Create query template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path requestBody: content: application/json: @@ -3493,9 +8302,44 @@ paths: datasetId: '{{DATASET}}' intervals: '{{STARTDATE}}/{{ENDDATE}}' limit: '{{LIMITS}}' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '359' + ETag: + schema: + type: string + example: W/"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 16:59:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3533,6 +8377,35 @@ paths: message: The query template has been saved successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '407' + ETag: + schema: + type: string + example: W/"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:05:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3549,6 +8422,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 62e18342-7e25-4122-8fca-6fb12fac3ff0 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID message: >- @@ -3566,6 +8440,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID_INPUT message: >- @@ -3574,6 +8449,35 @@ paths: trace: '' '409': description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '350' + ETag: + schema: + type: string + example: W/"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:03:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3587,6 +8491,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 18b6b123-4df5-4124-b6ec-73b667250e1c responseCode: CONFLICT + result: {} error: code: QUERY_TEMPLATE_ALREADY_EXISTS message: Template josnaks-aaa already exists @@ -3595,7 +8500,7 @@ paths: post: tags: - Query Templates - summary: List templates + summary: List Templates requestBody: content: application/json: @@ -3616,6 +8521,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '6864' + ETag: + schema: + type: string + example: W/"1ad0-xp24UiXXXiFWplmv5Acja7prSYM" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 13:46:03 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4176,17 +9110,17 @@ paths: created_date: '2024-04-25T06:31:20.819Z' updated_date: '2024-04-25T06:31:20.819Z' /v2/template/update/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template patch: tags: - Query Templates - summary: update template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + summary: Update Template requestBody: content: application/json: @@ -4207,6 +9141,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '314' + ETag: + schema: + type: string + example: W/"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:51:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4214,136 +9177,958 @@ paths: example: id: api.query.template.update ver: v2 - ts: '2024-05-02T13:16:31+05:30' + ts: '2024-05-10T11:21:47+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ab869754-d2f0-4b9a-b38b-04d9e5901b5d + resmsgid: 9e4a6959-0eb9-4fc4-8e6f-2eea534d1384 responseCode: OK result: message: Query template updated successfully templateId: sql11template1 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '429' + ETag: + schema: + type: string + example: W/"1ad-5sb8WUekFL8s4c1Ink6bUByoHho" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:53:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: required variables not exists to update' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:23:54+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 66b95cb3-2ef2-4735-9045-2674da552dbd + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Invalid template provided, A template should consist of + variables DATASET,STARTDATE,ENDDATE and type of json,sql + trace: '' + example-1: + summary: 'Failure: Template name validation failure' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:26:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: d6428fcf-53c9-465d-9431-769218f775b8 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Template name should contain alphanumeric characters and + single space between characters + trace: '' + example-2: + summary: 'Failure: query_type should when updating query' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:32:57+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c7a8675a-73f2-4764-abba-bfdf9f8b4621 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query_type when property query is present + trace: '' + example-3: + summary: 'Failure: query should present when updating query_type' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:34:55+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 2c1098b2-d7b3-4d39-98ee-e3e790fd23b4 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query when property query_type is present + trace: '' + /v2/template/query/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template + post: + tags: + - Query Templates + summary: Query Template + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.query + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + startdate: '2020-12-31' + enddate: '2024-12-31' + aggregationLevel: month + dataset: test + limit: 5 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-9hKB38iHEwYPT2MgF8puXcq05Ew" + Date: + schema: + type: string + example: Tue, 14 May 2024 06:22:24 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: JSON template with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:52:24+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + responseCode: OK + result: + - timestamp: '2023-09-01T00:00:00.000Z' + result: + school_id: 0 + example-1: + summary: 'Success: SQL template query with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:33:06+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '336' + ETag: + schema: + type: string + example: W/"150-T/XeSIt7PR7GcGEbET1e8n9zX7k" + Date: + schema: + type: string + example: Thu, 02 May 2024 07:29:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: query should present when query_type provided' + summary: 'Failure: invalid date range (native template)' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:30:36+05:30' + ts: '2024-05-02T12:59:14+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: f11f5287-77cd-43f0-b7f6-6d6b94b9cea2 + resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 responseCode: BAD_REQUEST + result: {} error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - #properties/request/dependencies should have property - query when property query_type is present + Invalid date range! make sure your range cannot be more + than 30 days trace: '' example-1: - summary: 'Failure: query_type should present when query is given' - value: - id: api.query.template.update - ver: v2 - ts: '2024-05-02T11:33:23+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 937cf790-867b-450b-afc9-86a96d02f636 - responseCode: BAD_REQUEST - error: - code: QUERY_TEMPLATE_INVALID_INPUT - message: >- - #properties/request/dependencies should have property - query_type when property query is present - trace: '' - example-2: - summary: 'Failure: template name validation failure' + summary: 'Failure: invalid date range' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:34:18+05:30' + ts: '2024-05-13T13:28:18+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9f3ae5aa-2e98-4204-8f87-b11e5bb3865a + resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 responseCode: BAD_REQUEST error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - Template name should contain alphanumeric characters and - single space between characters + Invalid date range! make sure your range cannot be more + than 30 days trace: '' - example-3: - summary: 'Failure: required template_id' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '357' + ETag: + schema: + type: string + example: W/"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U" + Date: + schema: + type: string + example: Mon, 13 May 2024 07:51:46 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Datasource not found in druid' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:38:16+05:30' + ts: '2024-05-13T13:21:46+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 94b71f80-bd1e-4283-8f3d-83833b60d08c - responseCode: BAD_REQUEST + resmsgid: b35a7050-b94c-4944-9630-233c9542272e + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_SOURCE_NOT_FOUND message: >- - #properties/request/required should have required - property 'template_id' + Dataset test with table hour is not available for + querying trace: '' - example-4: - summary: 'Failure: required variables not exists' + example-1: + summary: 'Failure: Datasource not found in live table' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T13:13:22+05:30' + ts: '2024-05-13T13:23:47+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dff3c22a-c198-4d1c-b87a-07d725f868f7 - responseCode: BAD_REQUEST + resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID - message: >- - Invalid template provided, A template should consist of - variables DATASET,STARTDATE,ENDDATE and type of json,sql + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource test11 not available for querying trace: '' - /v2/template/query/{template_id}: + /alerts/v1/notifications/search: + post: + tags: + - Alert Notification Channels + summary: Search Notificaation Channels + requestBody: + content: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/create: + post: + tags: + - Alert Notification Channels + summary: Configure channel + requestBody: + content: + application/json: + schema: + type: object + example: + manager: grafana + name: functional-metrics-email + type: email + config: + recipientAddresses: yravinderkumar33@gmail.com;ravinder@sanketika.in + subject: Obsrv Prod Alert + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert Notification Channels + summary: Publish Channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/test: + post: + tags: + - Alert Notification Channels + summary: Verify Configured Channel + requestBody: + content: + application/json: + schema: + type: object + example: + message: Testing Email integration. If you can read this, it's working! + payload: + error: + selectChannel: true + configureChannel: true + manager: grafana + name: udhw + type: email + config: + recipientAddresses: jerald@sanketika.in + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Notification Channels + summary: Update Specific Channel + requestBody: + content: + application/json: + schema: + type: object + example: + name: updated name + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Notification Channels + summary: Delete Channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/create: + post: + tags: + - Alert Silence + summary: Add Silence + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2024-01-11T07:56:23Z' + endDate: '2024-01-11T08:56:23Z' + alertId: c7464d32-1d8d-4eaf-9b23-1313a3ff8149 + manager: grafana + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert Silence + summary: Get Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Silence + summary: Delete Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/search: + get: + tags: + - Alert Silence + summary: Fetch All Silences + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Silence + summary: Modify Silence Request + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2023-08-09T10:50:59Z' + endDate: '2023-08-10T10:30:59Z' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alerts Wrapper + summary: 'Delete Configured Alert Rules ' + description: This URLwill provided access to user to delete any custom rule. + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alerts Wrapper + summary: 'Update Rule ' + description: >- + This URL will provide access to users to edit any properties in the + Alert rule and save the changes. + requestBody: + content: + application/json: + schema: + type: object + example: + description: >- + This alert rule is designed to promptly notify you when one or + more servers in your infrastructure become unresponsive or + inaccessible. + labels: + severity: warning + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'Publish Alert Rules ' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/create: + post: + tags: + - Alerts Wrapper + summary: 'Add Rule ' + requestBody: + content: + '*/*': + schema: + type: string + example: >- + "{\n \"name\": \"Total Api Calls1\",\n + \"manager\":\"grafana\",\n \"description\": \"This alert is + set up to notify you when the CPU usage on a host reaches a Low + level.\",\n \"expression\": \"(node_total_api_calls) > + 20\",\n \"category\": \"Infra\",\n + \"severity\":\"warning\",\n \"frequency\": \"1m\",\n + \"interval\": \"1m\",\n \"labels\": {\n \"component\": + \"api\",\n \"notificationChannel\": \"slack\"\n + },\n \"annotations\":{\n \"summary\":\"Host Low CPU + usage\"\n },\n \"metadata\": {\n \"query\": + [\n {\n \"refId\": + \"A\",\n \"datasourceUid\": + \"$datasourceUid\",\n \"queryType\": + \"\",\n \"relativeTimeRange\": + {\n \"from\": 600,\n + \"to\": 0\n },\n \"model\": + {\n \"refId\": \"A\",\n + \"hide\": false,\n \"editorMode\": + \"code\",\n \"expr\": + \"node_total_api_calls\",\n \"legendFormat\": + \"__auto\",\n \"range\": + true\n }\n },\n + {\n \"refId\": \"B\",\n + \"datasourceUid\": \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"B\",\n \"hide\": + false,\n \"type\": + \"reduce\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"B\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"reducer\": + \"last\",\n \"expression\": + \"A\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n },\n {\n \"refId\": + \"C\",\n \"datasourceUid\": + \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"C\",\n \"hide\": + false,\n \"type\": + \"threshold\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [\n + 20\n + ],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"C\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"expression\": + \"B\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n }\n ]\n }\n}" + parameters: + - name: Accept + in: header + schema: + type: string + example: application/json, text/plain, */* + - name: Accept-Language + in: header + schema: + type: string + example: en-GB,en + - name: Cache-Control + in: header + schema: + type: string + example: no-store + - name: Connection + in: header + schema: + type: string + example: keep-alive + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; + grafana_session=4eb514f6fef4ad6884e47e50254af650; + grafana_session_expiry=1687239735 + - name: Origin + in: header + schema: + type: string + example: http://localhost:3001 + - name: Pragma + in: header + schema: + type: string + example: no-store + - name: Referer + in: header + schema: + type: string + example: http://localhost:3001/alertRules/add + - name: Sec-Fetch-Dest + in: header + schema: + type: string + example: empty + - name: Sec-Fetch-Mode + in: header + schema: + type: string + example: cors + - name: Sec-Fetch-Site + in: header + schema: + type: string + example: same-origin + - name: Sec-GPC + in: header + schema: + type: integer + example: '1' + - name: User-Agent + in: header + schema: + type: string + example: >- + Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like + Gecko) Chrome/114.0.0.0 Safari/537.36 + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/search: post: tags: - - Query Templates - summary: query template - parameters: - - required: true + - Alerts Wrapper + summary: 'Search Alert Rule' + requestBody: + content: + application/json: + schema: + type: object + example: + request: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'Get Alert Rule' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/create: + post: + tags: + - Alert Metric Alias + summary: Create Custom Metric + requestBody: + content: + application/json: + schema: + type: object + example: + alias: Druid + component: MyTest + metric: sum(druid_supervisors{state='SUSPENDED'}) + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/search: + post: + tags: + - Alert Metric Alias + summary: List Create Metrics + requestBody: + content: + application/json: + schema: + type: object + example: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true schema: - title: template_id type: string - name: template_id + description: Unique identifier for the alert + patch: + tags: + - Alert Metric Alias + summary: Modify Metric + requestBody: + content: + application/json: + schema: + type: object + example: + alias: Druid native + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/delete/{alert_id}: + parameters: + - name: alert_id in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Metric Alias + summary: Delete Metric + responses: + '200': + description: Successful response + content: + application/json: {} + /v2/dataset/metrics: + post: + tags: + - Dataset metrics + summary: Dataset metrics requestBody: content: application/json: schema: type: object example: - id: api.query.template.query + id: api.dataset.metrics ver: v2 ts: '2024-04-10T16:10:50+05:30' params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - startdate: '2020-12-31' - enddate: '2024-12-31' - aggregationLevel: month - dataset: test - limit: 5 + dataset_id: test + category: + - data_freshness + - data_observability + - data_volume + - data_lineage + - connectors + - data_quality + query_time_period: 1 responses: '200': description: OK @@ -4353,148 +10138,222 @@ paths: type: object examples: example-0: - summary: 'Success: JSON template with request body' + summary: 'Success: Get all the data' value: - id: api.query.template.query + id: api.dataset.metrics ver: v2 - ts: '2024-05-14T11:52:24+05:30' + ts: '2025-01-15T15:24:21+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + resmsgid: ef86ff9c-4342-4cdc-a573-d469cbf436eb responseCode: OK result: - - timestamp: '2023-09-01T00:00:00.000Z' - result: - school_id: 0 + - category: data_freshness + status: Healthy + components: + - type: average_time_difference_in_min + threshold: 5 + value: 0 + status: Healthy + - type: freshness_query_time_in_min + threshold: 10 + value: 0 + status: Healthy + - category: data_observability + status: Unhealthy + components: + - type: data_observability_health + status: Unhealthy + - type: failure_percentage + value: 35.39823008849557 + - type: threshold_percentage + value: 5 + - type: importance_score + value: 0 + - category: data_volume + components: + - type: events_per_hour + value: 0 + - type: events_per_day + value: 95 + - type: events_per_n_day + value: 765 + - type: volume_percentage_by_hour + value: 0 + - type: volume_percentage_by_day + value: -87.58169934640523 + - type: volume_percentage_by_week + value: 0 + - type: growth_rate_percentage + value: 0 + - category: data_lineage + components: + - type: transformation_success + value: 765 + - type: dedup_success + value: 384 + - type: denormalization_success + value: 572 + - type: total_success + value: 1339 + - type: total_failed + value: 1034 + - type: transformation_failed + value: 1 + - type: dedup_failed + value: 574 + - type: denorm_failed + value: 193 + - category: connectors + components: + - id: failed + type: failed + value: 1608 + - id: api + type: success + value: 765 + - category: data_quality + components: + - type: incidents_failed + value: 1034 + - type: incidents_success + value: 1339 + - type: total_incidents + value: 2373 example-1: - summary: 'Success: SQL template query with request body' + summary: 'Success : Get particular category data' value: - id: api.query.template.query + id: api.dataset.metrics ver: v2 - ts: '2024-05-14T11:33:06+05:30' + ts: '2025-01-15T15:25:13+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + resmsgid: 10915ec3-a265-4502-bc9b-673562f94512 responseCode: OK result: - - __time: '2023-09-11T00:00:00.000Z' - school_category: secondary - gender: others - state_id: '15' - district_id: '2002' - block_id: '70' - cluster_id: '485' - obsrv.meta.source.connector: null - obsrv.meta.source.id: null - grade_sum: 18 - school_id_sum: 180378 - students_marked_sum: 12492 - students_present_sum: 2466 - total_count: 18 - total_students_sum: 12492 + - category: data_lineage + components: + - type: transformation_success + value: 765 + - type: dedup_success + value: 384 + - type: denormalization_success + value: 572 + - type: total_success + value: 1339 + - type: total_failed + value: 1034 + - type: transformation_failed + value: 1 + - type: dedup_failed + value: 574 + - type: denorm_failed + value: 193 '400': description: Bad Request content: application/json: schema: type: object - examples: - example-0: - summary: 'Failure: invalid date range (native template)' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-02T12:59:14+05:30' - params: - status: FAILED - resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' - example-1: - summary: 'Failure: invalid date range' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:28:18+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:23:22+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a6a26834-bb32-4e32-8dfd-473b0d21c8e8 + responseCode: BAD_REQUEST + error: + code: DATA_OUT_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + trace: '' '404': description: Not Found content: application/json: schema: type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:21:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b35a7050-b94c-4944-9630-233c9542272e - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: >- - Dataset test with table hour is not available for - querying - trace: '' - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:23:47+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: Datasource test11 not available for querying - trace: '' - /v2/dataset/copy: + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:15:30+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 25107bba-6d88-4e9b-b951-61e29a72b97a + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: Dataset with id test not found in live table + trace: '' + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:29:49+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 211553f9-ad91-4a4d-a6ba-d77d3e14b7ac + responseCode: FAILED + error: + code: FAILED_TO_FETCH_METRICS + message: Error while fetching metrics + trace: '' + /v2/data/analyze/pii: post: tags: - - Dataset copy - summary: Dataset copy + - Data Analyze PII + summary: Data Analyze PII requestBody: content: application/json: schema: type: object example: - id: api.datasets.copy - ver: '1.0' - ts: '2024-05-21T14:30:00Z' + id: api.data.analyze.pii + ver: v2 + ts: '1711966306164' params: - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 request: - source: - datasetId: test_dataset - isLive: true - destination: - datasetId: test-dataset-copy-1 + id: 268cb552-7de9-41cc-a413-a8bd3b64734c + dataset_id: taxi-payment-data + data: + - tripID: f0f5186b-3195-486e-9b79-0cd9da895707 + VendorID: '1' + tpep_pickup_datetime: '2024-10-31 00:46:40' + tpep_dropoff_datetime: '2024-10-31 00:53:20' + passenger_count: '1' + trip_distance: '1.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '151' + DOLocationID: '239' + payment_type: '1' + primary_passenger.email: Herminia_Veum@hotmail.com + primary_passenger.mobile: 200-329-2445 x3611 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Authorization + in: header + schema: + type: string + example: Bearer {{token}} responses: '200': description: OK @@ -4503,52 +10362,55 @@ paths: schema: type: object example: - id: api.dataset.copy + id: api.data.analyze.pii ver: v2 - ts: '2024-05-31T10:57:11+05:30' + ts: '2025-01-16T10:27:56+05:30' params: status: SUCCESS - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: 77080c18-67b3-489c-af7b-9739709e3c4b + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 8fd2cfe2-7912-40d1-8710-9b7da9a35bd9 responseCode: OK result: - dataset_id: sample1_copy.1 - message: Dataset clone successful - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:56:26+05:30' - params: - status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_EXISTS - message: Dataset with id sample1 already exists - trace: '' - '404': - description: Not Found + - field: primary_passenger.email + type: internet + score: 0.5 + reason: + - code: en + resourceKey: pii.descriptions.m014 + region: '' + score: 1 + - code: en + resourceKey: pii.descriptions.m004 + region: '' + score: 1 + - field: primary_passenger.mobile + type: phone + score: 0.5 + reason: + - code: en + resourceKey: pii.descriptions.m018 + region: USA + score: 1 + - code: en + resourceKey: pii.descriptions.m005 + region: '' + score: 1 + '500': + description: Internal Server Error content: application/json: schema: type: object example: - id: api.dataset.copy + id: api.data.analyze.pii ver: v2 - ts: '2024-05-31T10:59:59+05:30' + ts: '2025-01-16T10:29:58+05:30' params: status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: NOT_FOUND + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75aa71f9-ef84-41b9-ae1b-64cc7afd3ffb + responseCode: INTERNAL_SERVER_ERROR error: - code: DATASET_NOT_EXISTS - message: Dataset sample does not exists - trace: '' + code: ERR_BAD_REQUEST + message: Request failed with status code 401 + trace: '' \ No newline at end of file diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 74fc9cca..9e2c9802 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -10,6 +10,7 @@ metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" $ }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} @@ -27,6 +28,7 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + app.kubernetes.io/connector: {{ .Values.connector_id }} component: {{ printf "%s-%s" $jobName $component }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} @@ -87,6 +89,7 @@ metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" . }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} @@ -104,9 +107,9 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} - component: {{ printf "%s-%s" $jobName $component }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: - checksum/config: {{ .Files.Glob "cFonfigs/*" | toYaml | sha256sum }} + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} spec: serviceAccountName: {{ include "base.serviceaccountname" . }} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index ed51316d..ccabbc54 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -4,7 +4,7 @@ fullnameOverride: "" replicaCount: 1 -namespace: "flink" +namespace: "flink-connectors" commonLabels: system.processing: "true" release: monitoring @@ -246,6 +246,7 @@ serviceMonitor: # override flink_jobs # flink_jobs: +connector_id: "" commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index f22dcaef..14c3d9b1 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -1,9 +1,12 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "base.cronReleaseName" . }} + name: {{ .Release.Name }} namespace: {{ include "base.namespace" . }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/dataset: {{ .Values.dataset_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} + app.kubernetes.io/connector_instance: {{ .Values.instance_id }} {{- if .Values.commonAnnotations }} annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} @@ -19,6 +22,9 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 12 }} {{- end }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} + app.kubernetes.io/dataset: {{ .Values.dataset_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} + app.kubernetes.io/connector_instance: {{ .Values.instance_id }} spec: serviceAccountName: {{ .Values.serviceAccount.name }} restartPolicy: {{ .Values.restartPolicy }} @@ -70,7 +76,7 @@ spec: - | # Wait for the Spark pod to be ready SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') - kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master=local[*] --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" {{- end }} {{- with .Values.sidecars }} {{- toYaml . | nindent 12 }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index ceca090a..8069fe0e 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -22,10 +22,10 @@ commonAnnotations: {} podAnnotations: {} -podSecurityContext: {} - # runAsNonRoot: true - # runAsUser: 1001 - # fsGroup: 1001 +podSecurityContext: + runAsNonRoot: true + runAsUser: 1001 + fsGroup: 1001 securityContext: {} # readOnlyRootFilesystem: false @@ -142,6 +142,9 @@ instance_id: nyt-psql.1 main_class: org.sunbird.obsrv.connector.JDBCConnector main_file: jdbc-connector-1.0.0.jar +dataset_id: "" +connector_id: "" + ## Object Store Connector # technology: python # connector-source: object_store_connector-0.1.0 diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 3b6a82b9..5915a54d 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -5,7 +5,7 @@ from model.data_models import Action, ActionResponse, CommandPayload from service.db_service import DatabaseService from service.http_service import HttpService - +from copy import deepcopy class AlertManagerService(ICommand): @@ -95,17 +95,18 @@ def get_dataset_source_config(self, dataset_id: str) -> str: def get_modified_metric( self, service: str, metric: dict, payload: CommandPayload ) -> dict: + metric_data = deepcopy(metric) if service == "flink": substring = f"{payload.dataset_id}" modified_substring = substring.replace("-", "_") - modified_metric = metric["metric"].replace("dataset_id", modified_substring) - metric["metric"] = modified_metric - return metric + modified_metric = metric_data["metric"].replace("dataset_id", modified_substring) + metric_data["metric"] = modified_metric + return metric_data else: - metric["metric"] = metric["metric"].replace( + metric_data["metric"] = metric_data["metric"].replace( "dataset_id", payload.dataset_id ) - return metric + return metric_data def create_alert_metric( self, payload: CommandPayload, service: str, metric: dict, dataset_name: str diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 8f5b145e..b5073665 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -8,6 +8,9 @@ from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType from model.db_models import ConnectorInstance from service.db_service import DatabaseService +import time +from random import choice +from string import ascii_lowercase class ConnectorCommand(ICommand): @@ -38,15 +41,12 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"], active_connectors, dataset_id) + self._stop_connector_jobs(is_masterdata, dataset_id) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result - def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, dataset_id): - print(f"Uninstalling jobs for {namespace}..") - base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - + def _stop_connector_jobs(self, is_masterdata, dataset_id): # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -57,34 +57,66 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data # for release in masterdata_jar_config: # managed_releases.append(release["release_name"]) - helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - helm_ls_result = subprocess.run( - helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ## Clear all spark cronjobs for the dataset + spark_namespace = self.connector_job_config["spark"]["namespace"] + deployed_cron_jobs_cmd = [ + "kubectl", "get", "cronjobs", "-n", spark_namespace, + "--selector", f"app.kubernetes.io/dataset={dataset_id}", + "-o", "jsonpath=\"{.items[*].metadata.name}\"" + ] + + deployed_cron_jobs_result = subprocess.run( + deployed_cron_jobs_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode().splitlines()[1:] - job_names = {job.split()[0] for job in jobs if base_helm_chart in job} - spark_connector = {connector.id for connector in active_connectors if connector.connector_runtime == "spark"} - for release_name in spark_connector: - if release_name in job_names: - print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") - helm_uninstall_cmd = [ - "helm", - "uninstall", - release_name, - "--namespace", - namespace, - ] - helm_uninstall_result = subprocess.run( - helm_uninstall_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if helm_uninstall_result.returncode == 0: - print(f"Successfully uninstalled job '{release_name}'...") - else: - print(f"Error uninstalling job '{release_name}': {helm_uninstall_result.stderr.decode()}") - + + if deployed_cron_jobs_result.returncode == 0 and len(deployed_cron_jobs_result.stdout.decode().replace("\"", "").splitlines()): + jobs = deployed_cron_jobs_result.stdout.decode().replace("\"", "").splitlines()[0].split() + for job in jobs: + print(f"Uninstalling job {job} related to dataset'{dataset_id}'...") + helm_uninstall_cmd = [ + "helm", + "uninstall", + job, + "--namespace", + spark_namespace, + ] + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled job '{job}'...") + else: + print(f"Error uninstalling job '{job}': {helm_uninstall_result.stderr.decode()}") + + ## Clear all flink connectors that are not active + flink_namespace = self.connector_job_config["flink"]["namespace"] + registered_connectors = self._get_registered_connectors(runtime="flink") + print("Registered flink connectors: ", registered_connectors) + for connector in registered_connectors: + if connector[1] == 0: + ### Uninstall the helm chart + helm_uninstall_cmd = [ + "helm", + "uninstall", + connector[0].replace(".", "-").replace("_", "-"), + "--namespace", + flink_namespace, + ] + + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled deployment '{connector[0]}'...") + else: + print(f"Error uninstalling deployment '{connector}': {helm_uninstall_result.stderr.decode()}") + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None for connector in active_connectors: @@ -100,6 +132,9 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): ) break + if result is None: + result = ActionResponse(status="OK", status_code=200) + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -125,30 +160,30 @@ def _perform_flink_install(self, dataset_id, connector_instance): deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: - restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,app.kubernetes.io/component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,app.kubernetes.io/component={job_name}-taskmanager --namespace {namespace}".format( namespace=namespace, job_name=job_name ) print("Restart command: ", restart_cmd) # Run the helm command - helm_install_result = subprocess.run( + restart_cmd_result = subprocess.run( restart_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, ) - if helm_install_result.returncode == 0: + if restart_cmd_result.returncode == 0: print(f"Job {job_name} restart succeeded...") else: err = True + print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") return ActionResponse( status="ERROR", status_code=500, error_message="FLINK_HELM_LIST_EXCEPTION", ) - print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") if err is None: - result = ActionResponse(status="OK", status_code=200) + result = ActionResponse(status="OK", status_code=200) return result else: @@ -172,6 +207,8 @@ def _perform_flink_install(self, dataset_id, connector_instance): "--namespace", namespace, "--create-namespace", + "--set", "namespace={}".format(namespace), + "--set", "connector_id={}".format(connector_instance.connector_id), "--set-json", f"""flink_jobs={set_json_value.replace(" ", "")}""" ] @@ -201,8 +238,6 @@ def _perform_flink_install(self, dataset_id, connector_instance): result = ActionResponse(status="OK", status_code=200) return result - else: - self._stop_connector_jobs(is_masterdata=False, namespace="flink") else: print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") return ActionResponse( @@ -214,7 +249,10 @@ def _perform_flink_install(self, dataset_id, connector_instance): def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None - release_name = connector_instance.id + release_name = "{}-{}".format( + (dataset_id + "_" + connector_instance.connector_id).lower().replace("_", "-")[:54], + "".join(choice(ascii_lowercase) for i in range(6)) + ) connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] @@ -239,7 +277,11 @@ def _perform_spark_install(self, dataset_id, connector_instance): "--set", "technology={}".format(connector_instance.technology), "--set", - "instance_id={}".format(release_name), + "dataset_id={}".format(dataset_id), + "--set", + "connector_id={}".format(connector_instance.connector_id), + "--set", + "instance_id={}".format(connector_instance.id), "--set", "connector_source={}".format(connector_source["source"]), "--set", @@ -304,7 +346,6 @@ def _get_masterdata_details(self, dataset_id): return is_masterdata - ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False query = f""" @@ -321,48 +362,14 @@ def _get_live_instances(self, runtime, connector_instance): return has_live_instances - # def _perform_install(self, release): - # err = None - # result = None - # release_name = release["release_name"] - # helm_install_cmd = [ - # "helm", - # "upgrade", - # "--install", - # release_name, - # self.connector_job_chart_dir, - # "--namespace", - # self.connector_job_ns, - # "--create-namespace", - # "--set", - # "file.path={}".format(release["jar"]), - # "--set", - # "class.name={}".format(release["class"]), - # "--set", - # "job.name={}".format(release_name), - # "--set", - # "args={}".format(",".join(release["args"])), - # "--set", - # "schedule={}".format(release["schedule"]), - # ] - # helm_install_result = subprocess.run( - # helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - # ) - # if helm_install_result.returncode == 0: - # print(f"Job {release_name} deployment succeeded...") - # else: - # err = True - # result = ActionResponse( - # status="ERROR", - # status_code=500, - # error_message="FLINK_HELM_INSTALLATION_EXCEPTION", - # ) - # print( - # f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" - # ) - - # if err is None: - # result = ActionResponse(status="OK", status_code=200) - - # return result - + def _get_registered_connectors(self, runtime): + query = f""" + SELECT cr.id, COUNT(ci.id) AS instance_count + FROM connector_registry cr + LEFT JOIN connector_instances ci ON (cr.id = ci.connector_id AND ci.status = %s) + WHERE cr.runtime = %s + GROUP BY cr.id + """ + params = (DatasetStatusType.Live.name, runtime) + rows = self.db_service.execute_select_all(sql=query, params=params) + return rows \ No newline at end of file diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 40edf5f7..140d59bc 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -214,9 +214,6 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) - subprocess.run(["rm", "-rf", self.extraction_path]) - subprocess.run(["rm", "-rf", self.download_path]) - if not success: return RegistryResponse( status="failure", @@ -224,13 +221,16 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) result.append(registry_meta.to_dict()) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + return RegistryResponse( status="success", connector_info=result, message="Connectors registered successfully", statusCode=status.HTTP_200_OK ) - else: connector_id = ( self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 80276238..89cfa98e 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -175,7 +175,7 @@ connector_jobs: namespace: spark base_helm_chart: spark-connector-cron flink: - namespace: flink + namespace: flink-connectors base_helm_chart: flink-connector connector_registry: