From 9809511ad1ff8a4206bba0ebd71242046c378b62 Mon Sep 17 00:00:00 2001 From: Markham Lee Date: Fri, 1 Mar 2024 23:04:12 -0800 Subject: [PATCH] Small tweaks, fixing Env Vars --- .../finnhub_node/package.json | 4 +- .../finnhub_node/src/index.ts | 2 - etl_pipelines_nodejs/finnhub_node/src/test.js | 71 ------------------- .../finnhub_node/tsconfig.json | 2 +- .../finnhub_node/utils/utilities.js | 2 +- .../finnhub_node/utils/utilities.ts | 2 +- 6 files changed, 5 insertions(+), 78 deletions(-) delete mode 100644 etl_pipelines_nodejs/finnhub_node/src/test.js diff --git a/etl_pipelines_nodejs/finnhub_node/package.json b/etl_pipelines_nodejs/finnhub_node/package.json index 0fbd68d..f3f49ed 100644 --- a/etl_pipelines_nodejs/finnhub_node/package.json +++ b/etl_pipelines_nodejs/finnhub_node/package.json @@ -9,14 +9,14 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "engines": { - "node": "^18" + "node": "^19" }, "keywords": [], "author": "Markham Lee", "license": "MIT", "devDependencies": { "@types/express": "^4.17.21", - "@types/node": "^18.19.1", + "@types/node": "^18.19.21", "@typescript-eslint/eslint-plugin": "^7.1.0", "@typescript-eslint/parser": "^7.1.0", "eslint": "^8.57.0", diff --git a/etl_pipelines_nodejs/finnhub_node/src/index.ts b/etl_pipelines_nodejs/finnhub_node/src/index.ts index 20b57cd..b374d54 100644 --- a/etl_pipelines_nodejs/finnhub_node/src/index.ts +++ b/etl_pipelines_nodejs/finnhub_node/src/index.ts @@ -32,8 +32,6 @@ finnhubClient.quote(config.stock, (error: any, data: any, response: any) => { } - - }); //method to write data to InfluxDB diff --git a/etl_pipelines_nodejs/finnhub_node/src/test.js b/etl_pipelines_nodejs/finnhub_node/src/test.js deleted file mode 100644 index 7034b9c..0000000 --- a/etl_pipelines_nodejs/finnhub_node/src/test.js +++ /dev/null @@ -1,71 +0,0 @@ -// (C) Markham Lee 2023 - 2024 -// productivity-music-stocks-weather-IoT-dashboard -// https://github.com/MarkhamLee/productivity-music-stocks-weather-IoT-dashboard -// Node.js - TypeScript version of the Finnhub ETL: pulling down daily stock price data -// and writing it to InfluxDB. - -const finnhub = require('finnhub') -const { Point } = require('@influxdata/influxdb-client'); -const {config, createInfluxClient, sendSlackAlerts} = require("../utils/utilities") - -const api_key = finnhub.ApiClient.instance.authentications['api_key']; -api_key.apiKey = process.env['FINNHUB_KEY'] -const finnhubClient = new finnhub.DefaultApi() - - -finnhubClient.quote(config.stock, (error, data, response) => { - - if (error) { - console.error(error) - - } else { - console.log(data) - - const payload = { - "previous_close": Number(data['pc']), - "open": Number(data['o']), - "last_price": Number(data['l']), - "change": Number(data['dp']) - } - - writeData(payload) - - - } - - -}); - -//method to write data to InfluxDB -// the InfluxDB node.js library doesn't have a clean way of just -// pushing json data to the DB. So, the write methods will have to -// live in the primary ETL code for now. -const writeData = (payload) => { - - const bucket = config.bucket - const writeClient = createInfluxClient(bucket) - - let point = new Point(config.measurement) - .tag("Finnhub-API", "stock_prices",) - .floatField('change', payload.change) - .floatField('last_price', payload.last_price) - .floatField('open', payload.open) - .floatField('previous_close', payload.previous_close) - - // write data to InfluxDB - void setTimeout(() => { - - writeClient.writePoint(point); - console.log("Weather data successfully written to InfluxDB") - }, 1000) - - - // flush client - void setTimeout(() => { - - // flush InfluxDB client - writeClient.flush() - }, 1000) - - } - \ No newline at end of file diff --git a/etl_pipelines_nodejs/finnhub_node/tsconfig.json b/etl_pipelines_nodejs/finnhub_node/tsconfig.json index 1fd039b..2cac7df 100644 --- a/etl_pipelines_nodejs/finnhub_node/tsconfig.json +++ b/etl_pipelines_nodejs/finnhub_node/tsconfig.json @@ -17,4 +17,4 @@ "declarationMap": true, "lib": ["es2022"] } -} +} \ No newline at end of file diff --git a/etl_pipelines_nodejs/finnhub_node/utils/utilities.js b/etl_pipelines_nodejs/finnhub_node/utils/utilities.js index d2406f2..9919112 100644 --- a/etl_pipelines_nodejs/finnhub_node/utils/utilities.js +++ b/etl_pipelines_nodejs/finnhub_node/utils/utilities.js @@ -10,7 +10,7 @@ var influxdb_client_1 = require("@influxdata/influxdb-client"); var axios_1 = require("axios"); var config = { bucket: process.env.BUCKET, - finnhubKey: process.env.OPENWEATHER_KEY, + finnhubKey: process.env.FINNHUB_KEY, measurement: process.env.WEATHER_MEASUREMENT, org: process.env.INFLUX_ORG, stock: process.env.STOCK_SYMBOL, diff --git a/etl_pipelines_nodejs/finnhub_node/utils/utilities.ts b/etl_pipelines_nodejs/finnhub_node/utils/utilities.ts index e75f50d..728b3f4 100644 --- a/etl_pipelines_nodejs/finnhub_node/utils/utilities.ts +++ b/etl_pipelines_nodejs/finnhub_node/utils/utilities.ts @@ -24,7 +24,7 @@ const config: VarConfig = { bucket: process.env.BUCKET as string, finnhubKey: process.env.FINNHUB_KEY as string, - measurement: process.env.WEATHER_MEASUREMENT as string, + measurement: process.env.FINNHUB_MEASUREMENT_SPY as string, org: process.env.INFLUX_ORG as string, stock: process.env.STOCK_SYMBOL as string, token: process.env.INFLUX_KEY as string,