Minor refactoring, code clean-up

etl_pipelines/alpha_vantage_bonds_v2/main.py

@@ -1,8 +1,8 @@
 # Markham Lee (C) 2023
 # productivity-music-stocks-weather-IoT-dashboard
 # https://github.com/MarkhamLee/productivity-music-stocks-weather-IoT-dashboard
-# This pipeline retrieves t-bill rates for the last six days and then writes it
-# Postgres.
+# This pipeline retrieves t-bill rates for a given number of days and then
+# writes it to a Postgres database.
 import os
 import sys
 import requests
@@ -75,6 +75,7 @@ def parse_tbill_data(data: dict) -> object:
     # this also serves as our data validation step, the below steps won't work
     # if the data format is incorrect, fields are missing, etc.
 
+    # Get # of days of data to retrieve
     COUNT = int(os.environ['COUNT'])
 
     return utilities.bond_data_parser_entries(data, COUNT)
@@ -96,8 +97,7 @@ def write_data(connection: object, data: object,  table: str):
 
     else:
         logger.info(f"Postgres write successfuly, {response} rows written to database")  # noqa: E501
-
-    return status
+        return status
 
 
 def main():

etl_pipelines/finnhub/main.py

@@ -1,6 +1,6 @@
 # (C) Markham Lee 2023 - 2024
 # https://github.com/MarkhamLee/productivity-music-stocks-weather-IoT-dashboard
-# Retrieves current stock price for a given security and then writes the
+# Retrieves the current stock price for a given security and then writes the
 # data to InfluxDB.
 
 import os
@@ -31,7 +31,7 @@ def get_prices(symbol: str, *args):
 
     try:
         data = finn_util.get_stock_data(symbol, *args)
-        logger.info('stock price data retrieved')
+        logger.info(f'stock price data retrieved: {data}')
         return data
 
     except Exception as e:

etl_pipelines/github_dependabot/main.py

@@ -112,10 +112,10 @@ def main():
     MEASUREMENT = os.environ['GITHUB_ALERTS_MEASUREMENT']
     BUCKET = os.environ['DEVOPS_BUCKET']
 
-    tag_name = "Dependabot Alerts"
+    TAG_NAME = "Dependabot Alerts"
 
     # write data
-    github_utilities.write_github_data(count, MEASUREMENT, BUCKET, tag_name)
+    github_utilities.write_github_data(count, MEASUREMENT, BUCKET, TAG_NAME)
 
 
 if __name__ == "__main__":

etl_pipelines/github_dependabot/test.py

@@ -1,7 +1,6 @@
 # (C) Markham Lee 2023 - 2024
 # https://github.com/MarkhamLee/productivity-music-stocks-weather-IoT-dashboard
-# Test script for the pulling data from the GitHub Dependabot Alerts for
-# this repo
+# Test script for the pulling dependabot security alerts from the GitHub API
 
 import os
 import sys
@@ -69,7 +68,7 @@ def test_dependabot_alerts(self):
                              "Alert counting and data parsing failed")
         self.assertEqual(response, 0, "InfluxDB write unsuccessful")
 
-    # Check the response of the API call if the wrong key is passed
+    # Check the response of the API call if the wrong key is passed,
     # expected response is a 200 code from a successful Slack alert being
     # sent. I.e. you already know the bad key won't work, so what you want to
     # happen is the successful triggering of the Slack message.
@@ -83,6 +82,8 @@ def test_github_api_fail(self):
 
         self.assertEqual(data, 200, 'Bad API Key')
 
+    # Testing exception handling for sending bad data to
+    # InfluxDB.
     def test_db_write_exception_handlingl(self):
 
         # purposely send bad data that will fail InfluxDB's