[all] add stats/reporting metrics to /tmp/open5gs (#49)

README.md

@@ -14,3 +14,12 @@ open5gs has a limit of 16 different served TAIs hard-coded in OGS_MAX_NUM_OF_SER
 When configuring a node's PFCP connection to another node, you now have the option of adding `send_asr: false` to the configuration yaml. This bool (which defaults to `true`) indicates whether the node should send PFCP AssociationSetupRequests or not.
 
 The reason we need this bool/option is to turn it off for the CPS side. To allow KeyLTE routers to be seamlessly added to the existing CPS without requiring a reboot, we have to define all 256 served TAIs in the CPS configuration. Without this option, the CPS logs are full of errors trying to send ASR messages to KeyLTE routers that don't yet exist.
+
+## Metrics Exported
+We export metrics the same basic way the Linux Kernel exposes information in `/proc`: by writing variables as named files under the directory `/tmp/open5gs/`, and overwriting/updating the files as these values change over time. Reading the content of these files should always give you the most up-to-date value of the variable in question.
+
+The files `{service}_start_time` (e.g. `/tmp/open5gs/mme_start_time`) are comprised of two lines. The first line is the date-time parsed for human readibility (e.g. `Mon Jul 12 16:32:00 2021`) and the second line is the raw seconds since the Unix epoch (e.g. `1626107520`).
+
+Each service also has its own subdirectory (e.g. `/tmp/open5gs/mme/`) that contains exports other information/values. The files with a `num` in the front (e.g. `/tmp/open5gs/mme/num_enbs`) contain a single number that indicates how many of that item the service is connected to. The files with a `list` in front (e.g. `/tmp/open5gs/mme/list_enbs`) contains more information about each of those connected items. Each item has its own line (i.e. the number of lines in `list_enbs` should equal `num_enbs`), and each line has a series of key-value information separated by spaces (e.g. `ip:10.0.0.2 tac:2`).
+
+The reason I chose to use files is beacuse it's Unix-esque, it's completely agnostic to any specific network monitoring platform, and it should be easy to write a shim for whatever platform you want to integrate against. If you or your org have specific metrics you would like to see exposed, please reach out to me (or just issue a PR) and I will add them.

lib/core/ogs-log.c

@@ -29,6 +29,15 @@
 
 #include "ogs-core.h"
 
+// BEGIN SPENCERS FILE-LOG SYSTEM
+#include <dirent.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <time.h>
+#include <stdlib.h>
+// END SPENCERS FILE-LOG SYSTEM
+
 #define TA_NOR              "\033[0m"       /* all off */
 
 #define TA_FGC_BLACK        "\033[30m"      /* Black */
@@ -148,6 +157,55 @@ void ogs_log_cycle(void)
     }
 }
 
+// BEGIN SPENCERS FILE-LOG SYSTEM
+#define BASEFILE "/tmp/open5gs"
+
+void ogs_write_file_value(const char *filename, const char *value) {
+    FILE * fp;
+    struct stat st = {0};
+    char filestring[256];
+
+    strcpy(filestring, BASEFILE);
+    strcat(filestring, "/");
+    strcat(filestring, filename);
+
+    if (stat(BASEFILE, &st) == -1) {
+        mkdir(BASEFILE, 0744);
+    }
+
+    if ( (fp = fopen(filestring, "w")) == NULL) {
+        fprintf(stderr, "warning: could not open file %s\n", filename);
+        return;
+    }
+
+    fprintf(fp, "%s", value);
+    fclose(fp);
+    return;
+}
+
+void ogs_write_file_start(const char *filename) {
+    char buf[256];
+    time_t mytime = time(NULL);
+    sprintf(buf, "%s%d\n", ctime(&mytime), (int)mytime);
+    ogs_write_file_value(filename, buf);
+}
+
+void ogs_write_file_subdir(const char *filename) {
+    struct stat st = {0};
+    char filestring[256];
+
+    strcpy(filestring, BASEFILE);
+    strcat(filestring, "/");
+    strcat(filestring, filename);
+
+    if (stat(filestring, &st) == -1) {
+        mkdir(filestring, 0744);
+    }
+
+    return;
+}
+// END SPENCERS FILE-LOG SYSTEM
+
 ogs_log_t *ogs_log_add_stderr(void)
 {
     ogs_log_t *log = NULL;

lib/core/ogs-log.h

@@ -71,6 +71,12 @@ void ogs_log_init(void);
 void ogs_log_final(void);
 void ogs_log_cycle(void);
 
+// BEGIN SPENCERS FILE-LOG SYSTEM
+void ogs_write_file_value(const char *filename, const char *value);
+void ogs_write_file_start(const char *filename);
+void ogs_write_file_subdir(const char *filename);
+// END SPENCERS FILE-LOG SYSTEM
+
 ogs_log_t *ogs_log_add_stderr(void);
 ogs_log_t *ogs_log_add_file(const char *name);
 void ogs_log_remove(ogs_log_t *log);

lib/pfcp/context.c

@@ -2269,3 +2269,73 @@ void ogs_pfcp_pool_final(ogs_pfcp_sess_t *sess)
     ogs_pool_destroy(&sess->qer_id_pool);
     ogs_pool_destroy(&sess->bar_id_pool);
 }
+
+static char *stats_print_far(char *buf, ogs_pfcp_far_t *far) {
+    char buf1[OGS_ADDRSTRLEN];
+
+    if (far->apply_action & OGS_PFCP_APPLY_ACTION_DROP) {
+        buf += sprintf(buf, "act:DROP ");
+    } else if (far->apply_action & OGS_PFCP_APPLY_ACTION_FORW) {
+        buf += sprintf(buf, "act:FORW ");
+    } else if (far->apply_action & OGS_PFCP_APPLY_ACTION_BUFF) {
+        buf += sprintf(buf, "act:BUFF ");
+    } else {
+        buf += sprintf(buf, "act:%u ", far->apply_action);
+    }
+
+    switch (far->dst_if) {
+    case OGS_PFCP_INTERFACE_ACCESS:
+        buf += sprintf(buf, "dst_if:ACCESS ");
+        break;
+    case OGS_PFCP_INTERFACE_CORE:
+        buf += sprintf(buf, "dst_if:CORE ");
+        break;
+    case OGS_PFCP_INTERFACE_CP_FUNCTION:
+        buf += sprintf(buf, "dst_if:CP ");
+        break;
+    default:
+        buf += sprintf(buf, "dst_if:%u ", far->dst_if);
+    }
+
+    if (far->outer_header_creation.addr) {
+        buf += sprintf(buf, "dst_teid:0x%x dst_addr:%s ",
+            far->hash.f_teid.key.teid, OGS_INET_NTOP(&far->outer_header_creation.addr, buf1));
+    } else {
+        buf += sprintf(buf, "dst_teid:OGSTUN ");
+    }
+
+    return buf;
+}
+
+char *stats_print_pdr(char *buf, ogs_pfcp_pdr_t *pdr) {
+
+    buf += sprintf(buf, "\tpdr ");
+
+    switch (pdr->src_if) {
+    case OGS_PFCP_INTERFACE_ACCESS:
+        buf += sprintf(buf, "src_if:ACCESS ");
+        break;
+    case OGS_PFCP_INTERFACE_CORE:
+        buf += sprintf(buf, "src_if:CORE ");
+        break;
+    case OGS_PFCP_INTERFACE_CP_FUNCTION:
+        buf += sprintf(buf, "src_if:CP ");
+        break;
+    default:
+        buf += sprintf(buf, "src_if:%u ", pdr->src_if);
+    }
+
+    if (pdr->hash.teid.key != 0) {
+        buf += sprintf(buf, "src_teid:0x%x ", pdr->hash.teid.key);
+    } else {
+        buf += sprintf(buf, "src_teid:OGSTUN ");
+    }
+
+    if (pdr->far) {
+        buf = stats_print_far(buf, pdr->far);
+    } else {
+        buf += sprintf(buf, "far: NULL ");
+    }
+
+    return buf;
+}

lib/pfcp/context.h

@@ -488,6 +488,11 @@ ogs_pfcp_subnet_t *ogs_pfcp_find_subnet_by_dnn(int family, const char *dnn);
 void ogs_pfcp_pool_init(ogs_pfcp_sess_t *sess);
 void ogs_pfcp_pool_final(ogs_pfcp_sess_t *sess);
 
+#define MAX_FAR_STRING_LEN (38 + INET6_ADDRSTRLEN)
+#define MAX_PDR_STRING_LEN 38 + MAX_FAR_STRING_LEN
+
+char *stats_print_pdr(char *buf, ogs_pfcp_pdr_t *pdr);
+
 #ifdef __cplusplus
 }
 #endif

src/hss/hss-init.c

@@ -55,7 +55,9 @@ int hss_initialize(void)
 
     initialized = 1;
 
-    return OGS_OK;
+    ogs_write_file_start("hss_start_time");
+
+	return OGS_OK;
 }
 
 void hss_terminate(void)

src/main.c

@@ -77,6 +77,9 @@ static int check_signal(int signum)
         talloc_report_full(__ogs_talloc_core, stderr);
         break;
 
+    case SIGCHLD:
+	break;
+
     default:
         ogs_error("Signal-NUM[%d] received (%s)",
                 signum, ogs_signal_description_get(signum));

src/mme/esm-handler.c

@@ -170,6 +170,8 @@ int esm_handle_pdn_connectivity_request(mme_bearer_t *bearer,
         return OGS_ERROR;
     }
 
+    stats_update_mme_sessions();
+
     return OGS_OK;
 }
 
@@ -265,6 +267,8 @@ int esm_handle_information_response(mme_sess_t *sess,
         return OGS_ERROR;
     }
 
+    stats_update_mme_sessions();
+
     return OGS_OK;
 }
 

src/mme/mme-context.c

@@ -2808,6 +2808,8 @@ mme_enb_t *mme_enb_add(ogs_sock_t *sock, ogs_sockaddr_t *addr)
     ogs_info("[Added] Number of eNBs is now %d",
             ogs_list_count(&self.enb_list));
 
+    stats_update_mme_enbs();
+
     return enb;
 }
 
@@ -2842,6 +2844,8 @@ int mme_enb_remove(mme_enb_t *enb)
     ogs_info("[Removed] Number of eNBs is now %d",
             ogs_list_count(&self.enb_list));
 
+    stats_update_mme_enbs();
+
     return OGS_OK;
 }
 
@@ -3340,6 +3344,8 @@ mme_ue_t *mme_ue_add(enb_ue_t *enb_ue)
     ogs_info("[Added] Number of MME-UEs is now %d",
             ogs_list_count(&self.mme_ue_list));
 
+    stats_update_mme_ues();
+
     return mme_ue;
 }
 
@@ -3404,6 +3410,8 @@ void mme_ue_remove(mme_ue_t *mme_ue)
 
     ogs_info("[Removed] Number of MME-UEs is now %d",
             ogs_list_count(&self.mme_ue_list));
+
+    stats_update_mme_ues();
 }
 
 void mme_ue_remove_all(void)
@@ -3726,6 +3734,8 @@ int mme_ue_set_imsi(mme_ue_t *mme_ue, char *imsi_bcd)
 
     ogs_hash_set(self.imsi_ue_hash, mme_ue->imsi, mme_ue->imsi_len, mme_ue);
 
+    stats_update_mme_ues();
+
     return OGS_OK;
 }
 
@@ -4696,11 +4706,107 @@ static void stats_add_mme_session(void)
     mme_metrics_inst_global_inc(MME_METR_GLOB_GAUGE_MME_SESS);
     num_of_mme_sess = num_of_mme_sess + 1;
     ogs_info("[Added] Number of MME-Sessions is now %d", num_of_mme_sess);
+
+    stats_update_mme_sessions();
 }
 
 static void stats_remove_mme_session(void)
 {
     mme_metrics_inst_global_dec(MME_METR_GLOB_GAUGE_MME_SESS);
     num_of_mme_sess = num_of_mme_sess - 1;
     ogs_info("[Removed] Number of MME-Sessions is now %d", num_of_mme_sess);
+
+    stats_update_mme_sessions();
+}
+
+#define MAX_ENB_STRING_LEN (8 + OGS_MAX_IMSI_BCD_LEN + INET_ADDRSTRLEN + (8 * 4))
+
+void stats_update_mme_enbs(void)
+{
+    mme_enb_t *enb = NULL;
+
+    char buf[OGS_ADDRSTRLEN];
+    char *buffer = NULL;
+    char *ptr = NULL;
+    int i = 0;
+
+    char num[20];
+    sprintf(num, "%d\n", ogs_list_count(&self.enb_list));
+    ogs_write_file_value("mme/num_enbs", num);
+
+    ptr = buffer = ogs_malloc(MAX_ENB_STRING_LEN * ogs_list_count(&self.enb_list));
+    ogs_list_for_each(&self.enb_list, enb) {
+        ptr += sprintf(ptr, "ip:%s tac:%u",
+            OGS_ADDR(enb->sctp.addr, buf),enb->supported_ta_list[0].tac);
+        for(i = 1; i < enb->num_of_supported_ta_list; i++) {
+            ptr += sprintf(ptr, ",%u",enb->supported_ta_list[i].tac);
+        }
+        ptr += sprintf(ptr, "\n");
+    }
+
+    ogs_write_file_value("mme/list_enbs", buffer);
+    ogs_free(buffer);
+}
+
+#define MAX_UE_STRING_LEN (17 + OGS_MAX_IMSI_BCD_LEN + INET_ADDRSTRLEN + 3)
+
+void stats_update_mme_ues(void)
+{
+    mme_ue_t *mme_ue = NULL;
+    char buf1[OGS_ADDRSTRLEN];
+    char *buffer = NULL;
+    char *ptr = NULL;
+
+    char num[20];
+    sprintf(num, "%d\n", ogs_list_count(&self.mme_ue_list));
+    ogs_write_file_value("mme/num_ues", num);
+
+    ptr = buffer = ogs_malloc(MAX_UE_STRING_LEN * ogs_list_count(&self.mme_ue_list));
+    ogs_list_for_each(&self.mme_ue_list, mme_ue) {
+        ptr += sprintf(ptr, "imsi:%s enb:%s tac:%d\n", mme_ue->imsi_bcd,
+            mme_ue->enb_ue ? OGS_ADDR(mme_ue->enb_ue->enb->sctp.addr, buf1) : "",
+            mme_ue->tai.tac);
+    }
+    ogs_write_file_value("mme/list_ues", buffer);
+    ogs_free(buffer);
+}
+
+#define MAX_APN 63
+#define MAX_SESSION_STRING_LEN (21 + OGS_MAX_IMSI_BCD_LEN + MAX_APN + INET_ADDRSTRLEN + INET6_ADDRSTRLEN)
+
+void stats_update_mme_sessions(void)
+{
+    mme_ue_t *mme_ue = NULL;
+    mme_sess_t *sess = NULL;
+    ogs_session_t *session = NULL;
+
+    char buf1[OGS_ADDRSTRLEN];
+    char buf2[OGS_ADDRSTRLEN];
+    char buf3[OGS_ADDRSTRLEN];
+    char *buffer = NULL;
+    char *ptr = NULL;
+
+    char num[20];
+    sprintf(num, "%d\n", num_of_mme_sess);
+    ogs_write_file_value("mme/num_sessions", num);
+
+    ptr = buffer = ogs_malloc(MAX_SESSION_STRING_LEN * num_of_mme_sess);
+    ogs_list_for_each(&self.mme_ue_list, mme_ue) {
+        ogs_list_for_each(&mme_ue->sess_list, sess) {
+            ptr += sprintf(ptr, "imsi:%s enb:%s tac:%d ", mme_ue->imsi_bcd,
+                mme_ue->enb_ue ? OGS_ADDR(mme_ue->enb_ue->enb->sctp.addr, buf1) : "", 
+                mme_ue->tai.tac);
+            if (sess->session) {
+                session = sess->session;
+                ptr += sprintf(ptr, "apn:%s ip4:%s ip6:%s\n",
+                    session->name ? session->name : "",
+                    session->ue_ip.ipv4 ? OGS_INET_NTOP(session->ue_ip.addr, buf2) : "",
+                    session->ue_ip.ipv6 ? OGS_INET6_NTOP(session->ue_ip.addr6, buf3) : "");
+            } else {
+                ptr += sprintf(ptr, "apn: ip4: ip6:\n");
+            }
+        }
+    }
+    ogs_write_file_value("mme/list_sessions", buffer);
+    ogs_free(buffer);
 }

src/mme/mme-context.h

@@ -1018,6 +1018,10 @@ void mme_ebi_pool_clear(mme_ue_t *mme_ue);
 uint8_t mme_selected_int_algorithm(mme_ue_t *mme_ue);
 uint8_t mme_selected_enc_algorithm(mme_ue_t *mme_ue);
 
+void stats_update_mme_enbs(void);
+void stats_update_mme_ues(void);
+void stats_update_mme_sessions(void);
+
 #ifdef __cplusplus
 }
 #endif

src/mme/mme-init.c

@@ -84,6 +84,12 @@ int mme_initialize(void)
 
     initialized = 1;
 
+    ogs_write_file_start("mme_start_time");
+    ogs_write_file_subdir("mme");
+    stats_update_mme_enbs();
+    stats_update_mme_ues();
+    stats_update_mme_sessions();
+
     return OGS_OK;
 }