Refactoring, adding Unit tests

etl_pipelines_nodejs/finnhub_node/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+    preset: "ts-jest",
+    testEnvironment: "node",
+  };

etl_pipelines_nodejs/finnhub_node/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "tsc && node dist/index.js",
     "lint": "eslint . --ext .ts",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "jest"
   },
   "engines": {
     "node": "^18"
@@ -16,11 +16,14 @@
   "license": "MIT",
   "devDependencies": {
     "@types/express": "^4.17.21",
+    "@types/jest": "^29.5.12",
     "@types/node": "^18.19.21",
     "@typescript-eslint/eslint-plugin": "^7.1.0",
     "@typescript-eslint/parser": "^7.1.0",
     "eslint": "^8.57.0",
     "eslint-plugin-react": "^7.33.2",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.2",
     "typescript": "^5.3.3"
   },
   "dependencies": {

etl_pipelines_nodejs/finnhub_node/src/index.js

@@ -5,30 +5,40 @@
 // Node.js - TypeScript version of the Finnhub ETL: pulling down daily stock price data 
 // and writing it to InfluxDB.
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeData = exports.parseData = exports.getFinanceData = void 0;
 var finnhub = require('finnhub');
 var influxdb_client_1 = require("@influxdata/influxdb-client");
 var utilities_1 = require("../utils/utilities");
 var api_key = finnhub.ApiClient.instance.authentications['api_key'];
 api_key.apiKey = utilities_1.config.finnhubKey;
 var finnhubClient = new finnhub.DefaultApi();
-// get data from the Finnhub API via the Official Finnhub JS library
-finnhubClient.quote(utilities_1.config.stock, function (error, data, response) {
-    if (error) {
-        var message = "Pipeline failure for Node.js version of Finnhub Stock Price ETL, with error:";
-        var full_message = message.concat(error);
-        console.error(full_message);
-        (0, utilities_1.sendSlackAlerts)(full_message);
-    }
-    else {
-        console.log("Finnhub data received");
-        var payload = parseData(data);
-        writeData(payload);
-    }
-});
+var getFinanceData = function () {
+    // get data from the Finnhub API via the Official Finnhub JS library
+    finnhubClient.quote(utilities_1.config.stock, function (error, data, response) {
+        if (error) {
+            var message = "Pipeline failure for Node.js version of Finnhub Stock Price ETL, with error:";
+            var full_message = message.concat(error);
+            console.error(full_message);
+            (0, utilities_1.sendSlackAlerts)(full_message);
+            // exit process
+            return process.exit();
+        }
+        else {
+            console.log("Finnhub data received");
+            var payload = parseData(data);
+            writeData(payload);
+        }
+    });
+    return 0;
+};
+exports.getFinanceData = getFinanceData;
 // parse and validate the Finnhub data
 var parseData = function (data) {
     // validate data
-    (0, utilities_1.validateJson)(data);
+    var status = (0, utilities_1.validateJson)(data);
+    if (status == 1) {
+        return process.exit();
+    }
     var payload = {
         "previous_close": Number(data['pc']),
         "open": Number(data['o']),
@@ -38,6 +48,7 @@ var parseData = function (data) {
     console.log("InfluxDB payload ready", payload);
     return payload;
 };
+exports.parseData = parseData;
 // method to write data to InfluxDB
 // the InfluxDB node.js library doesn't have a clean way of just
 // pushing json data to the DB. So, the write methods will have to 
@@ -62,3 +73,4 @@ var writeData = function (payload) {
         writeClient.flush();
     }, 1000);
 };
+exports.writeData = writeData;

etl_pipelines_nodejs/finnhub_node/src/index.ts

@@ -12,31 +12,41 @@ const api_key = finnhub.ApiClient.instance.authentications['api_key']
 api_key.apiKey = config.finnhubKey 
 const finnhubClient = new finnhub.DefaultApi()
 
+const getFinanceData = () => {
+    // get data from the Finnhub API via the Official Finnhub JS library
+    finnhubClient.quote(config.stock, (error: any, data: any, response: any) => {
+
+            if (error) {
+                const message = "Pipeline failure for Node.js version of Finnhub Stock Price ETL, with error:"
+                const full_message = message.concat(error)
+                console.error(full_message)
+                sendSlackAlerts(full_message)
+                // exit process
+                return process.exit()
 
-// get data from the Finnhub API via the Official Finnhub JS library
-finnhubClient.quote(config.stock, (error: any, data: any, response: any) => {
-
-        if (error) {
-            const message = "Pipeline failure for Node.js version of Finnhub Stock Price ETL, with error:"
-            const full_message = message.concat(error)
-            console.error(full_message)
-            sendSlackAlerts(full_message)
-
-        } else {
-
-            console.log("Finnhub data received")
-            const payload = parseData(data)
-            writeData(payload)
+            } else {
 
-        }        
-    });
+                console.log("Finnhub data received")
+                const payload = parseData(data)
+                writeData(payload)
+
 
+            }        
+        });
+    return 0
+}
 
 // parse and validate the Finnhub data
 const parseData = (data: any) => {
 
     // validate data
-    validateJson(data)
+    const status = validateJson(data)
+
+    if (status == 1) {
+
+        return process.exit()
+
+    }
 
     const payload = {
         "previous_close": Number(data['pc']),
@@ -82,3 +92,5 @@ const writeData = (payload: any) => {
         }, 1000)
 
     }
+
+export {getFinanceData, parseData, writeData}

etl_pipelines_nodejs/finnhub_node/utils/utilities.js

@@ -37,13 +37,14 @@ var validateJson = function (data) {
     var validData = ajv.validate(finnhub_config_1.finnhubSchema, data);
     if (validData) {
         console.log("Data validation successful");
+        return 0;
     }
     else {
         var message = "Pipeline failure data validation - OpenWeather Air Quality (nodejs variant), exiting... ";
         console.error("Data validation error: ", ajv.errors);
         // exit the script so we don't attempt a DB write that won't work or
         // would write bad data to our db.
-        return process.exit();
+        return 1;
     }
 };
 exports.validateJson = validateJson;

etl_pipelines_nodejs/finnhub_node/utils/utilities.ts

@@ -48,14 +48,15 @@ const validateJson = (data: any) => {
         if (validData) {
 
             console.log("Data validation successful");
+            return 0
 
           } else {
 
             const message = "Pipeline failure data validation - OpenWeather Air Quality (nodejs variant), exiting... "
             console.error("Data validation error: ", ajv.errors);
             // exit the script so we don't attempt a DB write that won't work or
             // would write bad data to our db.
-            return process.exit()  
+            return 1
 
           }