Add monitoring metrics exporter to snowflake role

Add snowflake metrics exporter via prometheus node-exporter [1].
It extracts snowflake metrics from the snowflake journalctl logs
and exposes them via the node-exporter.

Metrics are available on the address http://<HOST_IP>:1900/metrics

[1] https://prometheus.io/docs/guides/node-exporter/

ansible/inventory/production/hosts

@@ -1,12 +1,18 @@
 $ANSIBLE_VAULT;1.1;AES256
-62343439313634643734343430323633663263613337613763343634656666306262663735336630
-3230643063323937323862373265306362646230653637380a303036653033643631623932303564
-37623632383336623030303064666466353266323935626564666265623964656534393538356162
-3866636137373966360a653761376461356435396132306631653363616164633764313561373533
-32666239303463323366636132383136346464626331303633383862653333633136376539386266
-32376162373765656465646132623336333835636431643134653136356264623636656266393632
-38646430306333373731366462346639376238323330626165623466613363343761353461633333
-35316331366663323730656461613333633466663237316630656331303964656231633964653436
-66326464393063373566353761306533346639366166663531323235323463393566343836356239
-32303861303962663932356538616530613034633834353766303431633935326434363335326561
-353166366163376631303666313730343532
+63343430313630366136666531373634356532393637663063356565653834666337316361363934
+6332313864323530333237653830373266396137333561630a323836363163663238653138306165
+65643531303737646639616638623137643837663639636134663066613335633361303738333032
+3166343866653530640a636432373063333537663139316334336662333533303234656638303234
+30653231643633373333333538663137303334643861303762346363333162363436643236623131
+31366533613932616463343538333035303865656633396362336463313633343362313364613533
+61336435653531316363383233336538376630616262353236346134656366616238306366373832
+62376561323161323334353761346237353634383531363661393163613731653964386564343965
+61393135386265303661653434613661313361646636393362653064393535376136356262666662
+31306235386531353931333131333939633166666162393265313238656636323632383432323065
+66626432653663336333646366663737653331396664616266303366393435383339623637323039
+32396634663731326331646534623837313933626235653530356230323763373032643930333330
+32373231323836343431633462633031646266636430623130323233653632633630646234333634
+61366665643330623339323362623861666662316130323863363039323234323439623166363131
+39636134316361633334633932363134386639353530386531663665353337643931623239313566
+38613963613137383231303130313539343836303637353735343864613262623733353831373038
+6131

ansible/roles/snowflake/files/export_metrics.sh

@@ -0,0 +1,2 @@
+sudo journalctl -o cat -u snowflake-proxy > /tmp/snowflake-logs.txt
+/usr/local/bin/snowflake2exporter.py --no-serve /tmp/snowflake-logs.txt  | sudo tee /var/lib/prometheus/node-exporter/snowflake.prom

ansible/roles/snowflake/files/snowflake2exporter.py

@@ -0,0 +1,179 @@
+#!/usr/bin/python3
+# This Script uses the following dependencies
+# pip install nums-from-string
+# pip install datetime
+#
+# To Run this script type:
+# python main.py <Log File Name>
+#
+# The default <Log File Name> is ./docker_snowflake.log
+#
+# Example:
+# python main.py snow.log
+#
+# Written By Allstreamer_
+# Licenced Under MIT
+#
+# Enhanced by MariusHerget
+# Further enhanced and modified by mrdrache333
+# Further enhanced by francisco-core
+
+import argparse
+import sys
+import re
+from datetime import datetime, timedelta
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+# Format of your timestamps in the beginning of the log
+# e.g. "2022/01/01 16:50:30 <LOG ENTRY>" => "%Y/%m/%d %H:%M:%S"
+TIMESTAMP_FORMAT = "%Y/%m/%d %H:%M:%S"
+
+def nums_from_string(string):
+    return [int(num) for num in re.findall(r"\d+", string)]
+
+class TextHandler(BaseHTTPRequestHandler):
+    logfile_path = None
+
+    def do_GET(self):
+
+        if self.path != "/metrics":
+            # If the request path is not /metrics, return a 404 Not Found error
+            self.send_error(404)
+            return
+        # Set the response status code to 200 OK
+        self.send_response(200)
+
+        # Set the content type to text/plain
+        self.send_header("Content-type", "text/plain")
+
+        # End the headers
+        self.end_headers()
+
+        # Return the metrics
+        print_stats(
+            self.logfile_path,
+            lambda x: self.wfile.write(x.encode())  # encode response
+        )
+
+
+def print_stats(logfile_path: str, printer_func):
+    # Read file
+    lines_all = readFile(logfile_path)
+
+    # Get the statistics for various time windows
+    # e.g. all time  => getDataFromLines(lines_all, 24)
+    # e.g. last 24h  => getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24))
+    # e.g. last Week => getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24 * 7))
+    stats = {
+        'All time': getDataFromLines(lines_all),
+        'Last 24h': getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24)),
+        'Last Week': getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24 * 7)),
+    }
+
+    # Print all the results in the Prometheus metric format
+    for time in stats:
+        stat = stats[time]
+        printer_func(
+            f"snowflake_served_people{{time=\"{time}\"}} {stat['connections']}\n" +
+            f"snowflake_upload_gb{{time=\"{time}\"}} {round(stat['upload_gb'], 4)}\n" +
+            f"snowflake_download_gb{{time=\"{time}\"}} {round(stat['download_gb'], 4)}\n"
+        )
+
+def readFile(logfile_path: str):
+    # Read in log file as lines
+    lines_all = []
+    with open(logfile_path, "r") as file:
+        lines_all = file.readlines()
+    return lines_all
+
+
+# Catchphrase for lines who do not start with a timestamp
+def catchTimestampException(rowSubString, timestampFormat):
+    try:
+        return datetime.strptime(rowSubString, timestampFormat)
+    except Exception:
+        return datetime.strptime("1970/01/01 00:00:00", "%Y/%m/%d %H:%M:%S")
+
+
+# Filter the log lines based on a time delta in hours
+def filterLinesBasedOnTimeDelta(log_lines, hours):
+    now = datetime.now()
+    length_timestamp_format = len(datetime.strftime(now, TIMESTAMP_FORMAT))
+    return filter(lambda row: now - timedelta(hours=hours) <= catchTimestampException(row[0:length_timestamp_format],
+                                                                                      TIMESTAMP_FORMAT) <= now,
+                  log_lines)
+
+
+# Convert traffic information (in B, KB, MB, or GB) to B (Bytes) and add up to a sum
+def get_byte_count(log_lines):
+    byte_count = 0
+    for row in log_lines:
+        symbols = row.split(" ")
+
+        # Use a dictionary to map units to their byte conversion values
+        units = {
+            "B": 1,
+            "KB": 1024,
+            "MB": 1024 * 1024,
+            "GB": 1024 * 1024 * 1024
+        }
+
+        # Use the dictionary to get the byte conversion value for the current unit
+        byte_count += int(symbols[1]) * units[symbols[2]]
+    return byte_count
+
+
+# Filter important lines from the log
+# Extract number of connections, uploaded traffic in GB and download traffic in GB
+def getDataFromLines(lines):
+    # Filter out important lines (Traffic information)
+    lines = [row.strip() for row in lines if "In the" in row]
+    lines = [row.split(",", 1)[1] for row in lines]
+
+    # Filter out all traffic log lines who did not had any connection
+    lines = [row for row in lines if nums_from_string(row)[0] != 0]
+
+    # Extract number of connections as a sum
+    connections = sum([nums_from_string(row)[0] for row in lines])
+
+    # Extract upload and download data
+    lines = [row.split("Relayed")[1] for row in lines]
+    upload = [row.split(",")[0].strip() for row in lines]
+    download = [row.split(",")[1].strip()[:-1] for row in lines]
+
+    # Convert upload/download data to GB
+    upload_gb = get_byte_count(upload) / 1024 / 1024 / 1024
+    download_gb = get_byte_count(download) / 1024 / 1024 / 1024
+
+    # Return information as a dictionary for better structure
+    return {'connections': connections, 'upload_gb': upload_gb, 'download_gb': download_gb}
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--serve",
+        dest="serve",
+        action="store_true",
+        help="Start http server directly on port 8080"
+    )
+    parser.add_argument(
+        "--no-serve",
+        dest="serve",
+        action="store_false",
+        help="Simply parse the input file"
+    )
+    parser.set_defaults(serve=True)
+
+    # Log file path from arguments (default: ./docker_snowflake.log)
+    parser.add_argument("logfile_path", default="./docker_snowflake.log")
+    args = parser.parse_args()
+
+    if args.serve:
+        # Start the HTTP server on port 8080
+        TextHandler.logfile_path = args.logfile_path
+        httpd = HTTPServer(("", 8080), TextHandler)
+        httpd.serve_forever()
+    else:
+        # Simply parse the file and print the resulting metrics
+        print_stats(args.logfile_path, sys.stdout.write)

ansible/roles/snowflake/tasks/main.yml

@@ -60,3 +60,6 @@
     that: "'unrestricted' in sf_nat_type.stdout"
     fail_msg: "[ERR] snowflake proxy NAT type is not unrestricted"
   tags: molecule-notest
+
+- name: Setup monitoring via prometheus node exporter
+  include_tasks: monitoring.yml

ansible/roles/snowflake/tasks/monitoring.yml

@@ -0,0 +1,35 @@
+---
+
+- name: ensure prometheus-node-exporter is installed
+  apt:
+    pkg: prometheus-node-exporter
+    state: present
+
+- name: Setup snowflake metrics creation scripts
+  copy:
+    src: "{{ role_path }}/files/{{ item }}"
+    dest: "/usr/local/bin/{{ item }}"
+    owner: root
+    group: root
+    mode: 0744
+  with_items:
+    - "snowflake2exporter.py"
+    - "export_metrics.sh"
+
+- name: Run CRON job to export snwoflake metrics to node-exporter
+  cron:
+    name: "snowflake_exporter"
+    user: "root"
+    weekday: "*"
+    minute: "*"
+    hour: "*"
+    job: "/usr/local/bin/export_metrics.sh"
+    state: present
+    cron_file: ansible_snowflake-export-metrics
+
+
+- name: Restart service cron to pick up config changes
+  ansible.builtin.systemd:
+    state: restarted
+    daemon_reload: true
+    name: cron