Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ndpiReader: add some global statistics about FPC #2680

Merged
merged 1 commit into from
Jan 17, 2025
Merged

ndpiReader: add some global statistics about FPC #2680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 74 additions & 7 deletions example/ndpiReader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ static char *bind_mask = NULL;
#define MAX_FARGS 64
static char* fargv[MAX_FARGS];
static int fargc = 0;
static int dump_fpc_stats = 0;

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../nDPI-custom/ndpiReader_defs.c"
Expand Down Expand Up @@ -716,6 +717,7 @@ static void help(u_int long_help) {
" --tls_heuristics | Enable TLS heuristics.\n"
" | It is a shortcut to --cfg=tls,dpi.heuristics,0x07\n"
" --cfg=proto,param,value | Configure the specific attribute of this protocol\n"
" --dump-fpc-stats | Print FPC statistics\n"
,
human_readeable_string_len,
min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
Expand Down Expand Up @@ -777,6 +779,7 @@ static void help(u_int long_help) {
#define OPTLONG_VALUE_OPENVPN_HEURISTICS 3001
#define OPTLONG_VALUE_TLS_HEURISTICS 3002
#define OPTLONG_VALUE_CONF 3003
#define OPTLONG_VALUE_FPC_STATS 3004

static struct option longopts[] = {
/* mandatory extcap options */
Expand Down Expand Up @@ -825,6 +828,7 @@ static struct option longopts[] = {
{ "openvpn_heuristics", no_argument, NULL, OPTLONG_VALUE_OPENVPN_HEURISTICS},
{ "tls_heuristics", no_argument, NULL, OPTLONG_VALUE_TLS_HEURISTICS},
{ "conf", required_argument, NULL, OPTLONG_VALUE_CONF},
{ "dump-fpc-stats", no_argument, NULL, OPTLONG_VALUE_FPC_STATS},

{0, 0, 0, 0}
};
Expand Down Expand Up @@ -1400,6 +1404,10 @@ static void parse_parameters(int argc, char **argv)
}
break;

case OPTLONG_VALUE_FPC_STATS:
dump_fpc_stats = 1;
break;

case OPTLONG_VALUE_CONF:
{
FILE *fd;
Expand Down Expand Up @@ -2401,7 +2409,7 @@ static void node_print_known_proto_walker(const void *node,
*/
static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) {
struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node;
u_int16_t thread_id = *((u_int16_t *) user_data), proto;
u_int16_t thread_id = *((u_int16_t *) user_data), proto, fpc_proto;

(void)depth;

Expand All @@ -2422,14 +2430,20 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept
process_ndpi_collected_info(ndpi_thread_info[thread_id].workflow, flow);

proto = flow->detected_protocol.proto.app_protocol ? flow->detected_protocol.proto.app_protocol : flow->detected_protocol.proto.master_protocol;

proto = ndpi_map_user_proto_id_to_ndpi_id(ndpi_thread_info[thread_id].workflow->ndpi_struct, proto);

fpc_proto = flow->fpc.proto.app_protocol ? flow->fpc.proto.app_protocol : flow->fpc.proto.master_protocol;
fpc_proto = ndpi_map_user_proto_id_to_ndpi_id(ndpi_thread_info[thread_id].workflow->ndpi_struct, fpc_proto);

ndpi_thread_info[thread_id].workflow->stats.protocol_counter[proto] += flow->src2dst_packets + flow->dst2src_packets;
ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes[proto] += flow->src2dst_bytes + flow->dst2src_bytes;
ndpi_thread_info[thread_id].workflow->stats.protocol_flows[proto]++;
ndpi_thread_info[thread_id].workflow->stats.flow_confidence[flow->confidence]++;
ndpi_thread_info[thread_id].workflow->stats.num_dissector_calls += flow->num_dissector_calls;
ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_counter[fpc_proto] += flow->src2dst_packets + flow->dst2src_packets;
ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_counter_bytes[fpc_proto] += flow->src2dst_bytes + flow->dst2src_bytes;
ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_flows[fpc_proto]++;
ndpi_thread_info[thread_id].workflow->stats.fpc_flow_confidence[flow->fpc.confidence]++;
}
}

Expand Down Expand Up @@ -3877,7 +3891,8 @@ static void printFlowsStats() {

num_flows = 0;
for(thread_id = 0; thread_id < num_threads; thread_id++) {
if(ndpi_thread_info[thread_id].workflow->stats.protocol_counter[0] > 0) {
if(ndpi_thread_info[thread_id].workflow->stats.protocol_counter[0] > 0 ||
(dump_fpc_stats && ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_counter[0] > 0)) {
for(i=0; i<NUM_ROOTS; i++)
ndpi_twalk(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i],
node_print_unknown_proto_walker, &thread_id);
Expand Down Expand Up @@ -3964,6 +3979,10 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
cumulative_stats.protocol_counter[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_counter[i];
cumulative_stats.protocol_counter_bytes[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_counter_bytes[i];
cumulative_stats.protocol_flows[i] += ndpi_thread_info[thread_id].workflow->stats.protocol_flows[i];

cumulative_stats.fpc_protocol_counter[i] += ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_counter[i];
cumulative_stats.fpc_protocol_counter_bytes[i] += ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_counter_bytes[i];
cumulative_stats.fpc_protocol_flows[i] += ndpi_thread_info[thread_id].workflow->stats.fpc_protocol_flows[i];
}

cumulative_stats.ndpi_flow_count += ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count;
Expand All @@ -3987,6 +4006,9 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
for(i = 0; i < sizeof(cumulative_stats.flow_confidence)/sizeof(cumulative_stats.flow_confidence[0]); i++)
cumulative_stats.flow_confidence[i] += ndpi_thread_info[thread_id].workflow->stats.flow_confidence[i];

for(i = 0; i < sizeof(cumulative_stats.fpc_flow_confidence)/sizeof(cumulative_stats.fpc_flow_confidence[0]); i++)
cumulative_stats.fpc_flow_confidence[i] += ndpi_thread_info[thread_id].workflow->stats.fpc_flow_confidence[i];

cumulative_stats.num_dissector_calls += ndpi_thread_info[thread_id].workflow->stats.num_dissector_calls;

/* LRU caches */
Expand Down Expand Up @@ -4132,6 +4154,14 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
(long long unsigned int)cumulative_stats.flow_confidence[i]);
}

if(dump_fpc_stats) {
for(i = 0; i < sizeof(cumulative_stats.fpc_flow_confidence)/sizeof(cumulative_stats.fpc_flow_confidence[0]); i++) {
if(cumulative_stats.fpc_flow_confidence[i] != 0)
printf("\tFPC Confidence: %-10s %-13llu (flows)\n", ndpi_fpc_confidence_get_name(i),
(long long unsigned int)cumulative_stats.fpc_flow_confidence[i]);
}
}

if(dump_internal_stats) {
char buf[1024];

Expand Down Expand Up @@ -4233,6 +4263,15 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
(long long unsigned int)cumulative_stats.flow_confidence[i]);
}

if(dump_fpc_stats) {
for(i = 0; i < sizeof(cumulative_stats.fpc_flow_confidence)/sizeof(cumulative_stats.fpc_flow_confidence[0]); i++) {
if(cumulative_stats.fpc_flow_confidence[i] != 0)
fprintf(results_file, "FPC Confidence %-17s: %llu (flows)\n",
ndpi_fpc_confidence_get_name(i),
(long long unsigned int)cumulative_stats.fpc_flow_confidence[i]);
}
}

if(dump_internal_stats) {
char buf[1024];

Expand Down Expand Up @@ -4317,30 +4356,58 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i));

if(cumulative_stats.protocol_counter[i] > 0) {
if(cumulative_stats.protocol_counter[i] > 0 ||
(dump_fpc_stats && cumulative_stats.fpc_protocol_counter[i] > 0)) {
breed_stats_bytes[breed] += (long long unsigned int)cumulative_stats.protocol_counter_bytes[i];
breed_stats_pkts[breed] += (long long unsigned int)cumulative_stats.protocol_counter[i];
breed_stats_flows[breed] += (long long unsigned int)cumulative_stats.protocol_flows[i];

if(results_file)
fprintf(results_file, "%s\t%llu\t%llu\t%u\n",
if(results_file) {
fprintf(results_file, "%s\t%llu\t%llu\t%u",
ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i)),
(long long unsigned int)cumulative_stats.protocol_counter[i],
(long long unsigned int)cumulative_stats.protocol_counter_bytes[i],
cumulative_stats.protocol_flows[i]);
if(dump_fpc_stats) {
fprintf(results_file, "\t%llu\t%llu\t%u",
(long long unsigned int)cumulative_stats.fpc_protocol_counter[i],
(long long unsigned int)cumulative_stats.fpc_protocol_counter_bytes[i],
cumulative_stats.fpc_protocol_flows[i]);
if(cumulative_stats.protocol_counter[i] != cumulative_stats.fpc_protocol_counter[i] ||
cumulative_stats.protocol_counter_bytes[i] != cumulative_stats.fpc_protocol_counter_bytes[i] ||
cumulative_stats.protocol_flows[i] != cumulative_stats.fpc_protocol_flows[i])
fprintf(results_file, "\t(*)");
}
fprintf(results_file, "\n");
}

if(!quiet_mode) {
printf("\t%-20s packets: %-13llu bytes: %-13llu "
"flows: %-13u\n",
"flows: %-13u",
ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i)),
(long long unsigned int)cumulative_stats.protocol_counter[i],
(long long unsigned int)cumulative_stats.protocol_counter_bytes[i],
cumulative_stats.protocol_flows[i]);
if(dump_fpc_stats) {
printf(" FPC packets: %-13llu FPC bytes: %-13llu "
"FPC flows: %-13u",
(long long unsigned int)cumulative_stats.fpc_protocol_counter[i],
(long long unsigned int)cumulative_stats.fpc_protocol_counter_bytes[i],
cumulative_stats.fpc_protocol_flows[i]);
if(cumulative_stats.protocol_counter[i] != cumulative_stats.fpc_protocol_counter[i] ||
cumulative_stats.protocol_counter_bytes[i] != cumulative_stats.fpc_protocol_counter_bytes[i] ||
cumulative_stats.protocol_flows[i] != cumulative_stats.fpc_protocol_flows[i])
printf("(*)");
}
printf("\n");
}
}
}
if(!quiet_mode && dump_fpc_stats) {
printf("\n\tNOTE: protocols with different standard and FPC statistics are marked\n");
}

if(!quiet_mode) {
printf("\n\nProtocol statistics:\n");
Expand Down
4 changes: 4 additions & 0 deletions example/reader_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@ typedef struct ndpi_stats {
u_int64_t protocol_counter[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
u_int64_t protocol_counter_bytes[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
u_int32_t protocol_flows[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
u_int64_t fpc_protocol_counter[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
u_int64_t fpc_protocol_counter_bytes[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
u_int32_t fpc_protocol_flows[NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + 1];
u_int32_t ndpi_flow_count;
u_int32_t flow_count[3];
u_int64_t tcp_count, udp_count;
Expand All @@ -367,6 +370,7 @@ typedef struct ndpi_stats {
u_int16_t max_packet_len;
u_int64_t dpi_packet_count[3];
u_int64_t flow_confidence[NDPI_CONFIDENCE_MAX];
u_int64_t fpc_flow_confidence[NDPI_FPC_CONFIDENCE_MAX];
u_int64_t num_dissector_calls;

struct ndpi_lru_cache_stats lru_stats[NDPI_LRUCACHE_MAX];
Expand Down
1 change: 1 addition & 0 deletions tests/cfgs/fpc/config.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
--dump-fpc-stats
1 change: 1 addition & 0 deletions tests/cfgs/fpc/pcap/1kxun.pcap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../default/pcap/1kxun.pcap
1 change: 1 addition & 0 deletions tests/cfgs/fpc/pcap/signal_videocall.pcapng
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../default/pcap/signal_videocall.pcapng
Loading
Loading