From c12006c7e26a888305af690ac281cae99e4a4637 Mon Sep 17 00:00:00 2001 From: Ivan Nardi Date: Tue, 11 Feb 2025 12:43:58 +0100 Subject: [PATCH] DNS: evaluate all flow risks even if sub-classification is disabled --- doc/configuration_parameters.md | 2 +- src/include/ndpi_api.h | 4 +- src/lib/ndpi_main.c | 14 ++--- src/lib/protocols/dns.c | 59 +++++++++---------- src/lib/protocols/fastcgi.c | 2 +- src/lib/protocols/http.c | 2 +- src/lib/protocols/quic.c | 2 +- .../result/dns.pcap.out | 2 +- .../dns_subclassification_enable/config.txt | 1 + .../pcap/dns.pcap | 1 + .../result/dns.pcap.out | 29 +++++++++ .../result/anydesk.pcapng.out | 2 +- .../result/dns.pcap.out | 2 +- 13 files changed, 75 insertions(+), 47 deletions(-) create mode 100644 tests/cfgs/dns_subclassification_enable/config.txt create mode 120000 tests/cfgs/dns_subclassification_enable/pcap/dns.pcap create mode 100644 tests/cfgs/dns_subclassification_enable/result/dns.pcap.out diff --git a/doc/configuration_parameters.md b/doc/configuration_parameters.md index efffac2340b..bb1e17a675b 100644 --- a/doc/configuration_parameters.md +++ b/doc/configuration_parameters.md @@ -67,7 +67,7 @@ List of the supported configuration options: | "stun" | "metadata.attribute.relayed_address" | enable | NULL | NULL | Enable/disable extraction of (xor-)relayed-address attribute for STUN flows. If it is disabled, STUN classification might be significant faster | | "stun" | "metadata.attribute.peer_address" | enable | NULL | NULL | Enable/disable extraction of (xor-)peer-address attribute for STUN flows. If it is disabled, STUN classification might be significant faster; however sub-classification capability might be negatively impacted | | "bittorrent" | "metadata.hash" | enable | NULL | NULL | Enable/disable extraction of hash metadata for Bittorrent flows. | -| "dns" | "subclassification" | enable | NULL | NULL | Enable/disable sub-classification of DNS flows (via query/response domain name). If disabled, some flow risks are not checked | +| "dns" | "subclassification" | enable | NULL | NULL | Enable/disable sub-classification of DNS flows (via query/response domain name). | | "dns" | "process_response" | enable | NULL | NULL | Enable/disable processing of DNS responses. By default, DNS flows are fully classified after the first request/response pair (or after the first response, if the request is missing). If this parameter is disabled, the flows are fully classified after the first packet, i.e. usually after the first request; in that case, some flow risks are not checked and some metadata are not exported | | "http" | "process_response" | enable | NULL | NULL | Enable/disable processing of HTTP responses. By default, HTTP flows are usually fully classified after the first request/response pair. If this parameter is disabled, the flows are fully classified after the first request (or after the first response, if the request is missing); in that case, some flow risks are not checked and some metadata are not exported | | "http" | "subclassification" | enable | NULL | NULL | Enable/disable sub-classification of HTTP flows | diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 72dfe82a9ff..3d95f600729 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -488,6 +488,7 @@ extern "C" { * @par string_to_match_len = the length of the string * @par ret_match = completed returned match information * @par master_protocol_id = value of the ID associated to the master protocol detected + * @par update_flow_classification = update or not protocol (sub)classification * @return the ID of the matched subprotocol * */ @@ -496,7 +497,8 @@ extern "C" { char *string_to_match, u_int string_to_match_len, ndpi_protocol_match_result *ret_match, - u_int16_t master_protocol_id); + u_int16_t master_protocol_id, + int update_flow_classification); /** * Check if the string content passed match with a protocol diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 39c92487467..801a9d38f73 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -10413,7 +10413,8 @@ u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_ struct ndpi_flow_struct *flow, char *string_to_match, u_int string_to_match_len, ndpi_protocol_match_result *ret_match, - u_int16_t master_protocol_id) { + u_int16_t master_protocol_id, + int update_flow_classification) { u_int16_t rc; ndpi_protocol_category_t id; @@ -10421,7 +10422,7 @@ u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_ memset(ret_match, 0, sizeof(*ret_match)); - rc = ndpi_automa_match_string_subprotocol(ndpi_str, flow, + rc = ndpi_automa_match_string_subprotocol(ndpi_str, update_flow_classification ? flow : NULL, string_to_match, string_to_match_len, master_protocol_id, ret_match); id = ret_match->protocol_category; @@ -10430,13 +10431,12 @@ u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_ string_to_match_len, &id) != -1) { /* if(id != -1) */ { ret_match->protocol_category = id; - if(flow) - flow->category = id; + flow->category = id; rc = master_protocol_id; } } - if(flow && ndpi_str->risky_domain_automa.ac_automa != NULL) { + if(ndpi_str->risky_domain_automa.ac_automa != NULL) { u_int32_t proto_id; u_int16_t rc1 = ndpi_match_string_common(ndpi_str->risky_domain_automa.ac_automa, string_to_match, string_to_match_len, @@ -10450,7 +10450,7 @@ u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_ } /* Add punycode check */ - if(flow && ndpi_check_punycode_string(string_to_match, string_to_match_len)) { + if(ndpi_check_punycode_string(string_to_match, string_to_match_len)) { char str[64] = { '\0' }; strncpy(str, string_to_match, ndpi_min(string_to_match_len, sizeof(str)-1)); @@ -10477,7 +10477,7 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc what = name, what_len = name_len; subproto = ndpi_match_host_subprotocol(ndpi_struct, flow, what, what_len, - &ret_match, master_protocol); + &ret_match, master_protocol, 1); if(subproto != NDPI_PROTOCOL_UNKNOWN) { ndpi_set_detected_protocol(ndpi_struct, flow, subproto, master_protocol, NDPI_CONFIDENCE_DPI); diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index 70e1b81f5bd..19215e79b67 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -801,44 +801,39 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st } } - if(len > 0) { - if(ndpi_struct->cfg.dns_subclassification_enabled || ndpi_struct->cfg.fpc_enabled) { - ndpi_protocol_match_result ret_match; - - /* Avoid writing on flow (i.e. updating classification) if subclassification is disabled */ - ret.proto.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, ndpi_struct->cfg.dns_subclassification_enabled ? flow : NULL, - flow->host_server_name, - strlen(flow->host_server_name), - &ret_match, - NDPI_PROTOCOL_DNS); - /* Add to FPC DNS cache */ - if(ndpi_struct->cfg.fpc_enabled && - ret.proto.app_protocol != NDPI_PROTOCOL_UNKNOWN && - ret.proto.app_protocol != NDPI_PROTOCOL_DNS && - (flow->protos.dns.rsp_type == 0x1 || flow->protos.dns.rsp_type == 0x1c) && /* A, AAAA */ - ndpi_struct->fpc_dns_cache) { - ndpi_lru_add_to_cache(ndpi_struct->fpc_dns_cache, - fpc_dns_cache_key_from_dns_info(flow), ret.proto.app_protocol, - ndpi_get_current_time(flow)); - } + if(strlen(flow->host_server_name) > 0) { + ndpi_protocol_match_result ret_match; + + /* Avoid updating classification if subclassification is disabled */ + ret.proto.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow, + flow->host_server_name, + strlen(flow->host_server_name), + &ret_match, + NDPI_PROTOCOL_DNS, + ndpi_struct->cfg.dns_subclassification_enabled ? 1 : 0); + /* Add to FPC DNS cache */ + if(ndpi_struct->cfg.fpc_enabled && + ret.proto.app_protocol != NDPI_PROTOCOL_UNKNOWN && + ret.proto.app_protocol != NDPI_PROTOCOL_DNS && + (flow->protos.dns.rsp_type == 0x1 || flow->protos.dns.rsp_type == 0x1c) && /* A, AAAA */ + ndpi_struct->fpc_dns_cache) { + ndpi_lru_add_to_cache(ndpi_struct->fpc_dns_cache, + fpc_dns_cache_key_from_dns_info(flow), ret.proto.app_protocol, + ndpi_get_current_time(flow)); + } - if(!ndpi_struct->cfg.dns_subclassification_enabled) - ret.proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; + if(!ndpi_struct->cfg.dns_subclassification_enabled) + ret.proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; - if(ret.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN) - ret.proto.master_protocol = checkDNSSubprotocol(s_port, d_port); - else - ret.proto.master_protocol = NDPI_PROTOCOL_DNS; + if(ret.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN) + ret.proto.master_protocol = checkDNSSubprotocol(s_port, d_port); + else + ret.proto.master_protocol = NDPI_PROTOCOL_DNS; - ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0); - } else { - ret.proto.master_protocol = checkDNSSubprotocol(s_port, d_port); - ret.proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; - } + ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0); /* Category is always NDPI_PROTOCOL_CATEGORY_NETWORK, regardless of the subprotocol */ flow->category = NDPI_PROTOCOL_CATEGORY_NETWORK; - } /* Report if this is a DNS query or reply */ diff --git a/src/lib/protocols/fastcgi.c b/src/lib/protocols/fastcgi.c index 52518b0c960..484d98ed62b 100644 --- a/src/lib/protocols/fastcgi.c +++ b/src/lib/protocols/fastcgi.c @@ -210,7 +210,7 @@ static void ndpi_search_fastcgi(struct ndpi_detection_module_struct *ndpi_struct ndpi_match_host_subprotocol(ndpi_struct, flow, flow->host_server_name, strlen(flow->host_server_name), - &ret_match, NDPI_PROTOCOL_FASTCGI); + &ret_match, NDPI_PROTOCOL_FASTCGI, 1); ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0); if(ndpi_is_valid_hostname((char *)packet->host_line.ptr, diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index f69d36f7cfa..cb3376e8e3b 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -571,7 +571,7 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp origin_hostname, origin_hostname_len, &ret_match, - master_protocol); + master_protocol, 1); } } } diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c index e7b21786241..0ddc1a83037 100644 --- a/src/lib/protocols/quic.c +++ b/src/lib/protocols/quic.c @@ -1460,7 +1460,7 @@ void process_chlo(struct ndpi_detection_module_struct *ndpi_struct, ndpi_match_host_subprotocol(ndpi_struct, flow, flow->host_server_name, strlen(flow->host_server_name), - &ret_match, NDPI_PROTOCOL_QUIC); + &ret_match, NDPI_PROTOCOL_QUIC, 1); flow->protos.tls_quic.client_hello_processed = 1; /* Allow matching of custom categories */ ndpi_check_dga_name(ndpi_struct, flow, diff --git a/tests/cfgs/dns_subclassification_and_process_response_disable/result/dns.pcap.out b/tests/cfgs/dns_subclassification_and_process_response_disable/result/dns.pcap.out index a9f20999bf8..db13bf43365 100644 --- a/tests/cfgs/dns_subclassification_and_process_response_disable/result/dns.pcap.out +++ b/tests/cfgs/dns_subclassification_and_process_response_disable/result/dns.pcap.out @@ -9,7 +9,7 @@ LRU cache mining: 0/0/0 (insert/search/found) LRU cache msteams: 0/0/0 (insert/search/found) LRU cache fpc_dns: 0/0/0 (insert/search/found) Automa host: 2/2 (search/found) -Automa domain: 0/0 (search/found) +Automa domain: 2/0 (search/found) Automa tls cert: 0/0 (search/found) Automa risk mask: 2/0 (search/found) Automa common alpns: 0/0 (search/found) diff --git a/tests/cfgs/dns_subclassification_enable/config.txt b/tests/cfgs/dns_subclassification_enable/config.txt new file mode 100644 index 00000000000..76280fbb3bd --- /dev/null +++ b/tests/cfgs/dns_subclassification_enable/config.txt @@ -0,0 +1 @@ +--cfg=dns,subclassification,1 diff --git a/tests/cfgs/dns_subclassification_enable/pcap/dns.pcap b/tests/cfgs/dns_subclassification_enable/pcap/dns.pcap new file mode 120000 index 00000000000..aea7db12bf5 --- /dev/null +++ b/tests/cfgs/dns_subclassification_enable/pcap/dns.pcap @@ -0,0 +1 @@ +../../default/pcap/dns.pcap \ No newline at end of file diff --git a/tests/cfgs/dns_subclassification_enable/result/dns.pcap.out b/tests/cfgs/dns_subclassification_enable/result/dns.pcap.out new file mode 100644 index 00000000000..7e63e6d78f3 --- /dev/null +++ b/tests/cfgs/dns_subclassification_enable/result/dns.pcap.out @@ -0,0 +1,29 @@ +DPI Packets (UDP): 3 (1.50 pkts/flow) +Confidence DPI : 2 (flows) +Num dissector calls: 2 (1.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache fpc_dns: 1/0/0 (insert/search/found) +Automa host: 3/3 (search/found) +Automa domain: 3/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 1/0 (search/found) +Automa common alpns: 0/0 (search/found) +Patricia risk mask: 2/0 (search/found) +Patricia risk mask IPv6: 0/0 (search/found) +Patricia risk: 1/0 (search/found) +Patricia risk IPv6: 0/0 (search/found) +Patricia protocols: 4/0 (search/found) +Patricia protocols IPv6: 0/0 (search/found) + +Google 3 226 1 +WhatsApp 2 310 1 + +Acceptable 5 536 2 + + 1 UDP 82.178.113.245:47255 <-> 82.178.158.181:53 [VLAN: 785][proto: 5.142/DNS.WhatsApp][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5.142/DNS.WhatsApp, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/219 bytes][Goodput ratio: 36/73][0.00 sec][Hostname/SNI: e7.whatsapp.net][169.45.219.235][PLAIN TEXT (whatsapp)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 2 UDP 192.168.170.20:53 <-> 192.168.170.8:32795 [proto: 5.126/DNS.Google][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5.126/DNS.Google, Confidence: DPI][DPI packets: 1][cat: Network/14][2 pkts/151 bytes <-> 1 pkts/75 bytes][Goodput ratio: 44/43][41.07 sec][Hostname/SNI: www.l.google.com][0.0.0.0][PLAIN TEXT (google)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/cfgs/subclassification_disable/result/anydesk.pcapng.out b/tests/cfgs/subclassification_disable/result/anydesk.pcapng.out index e2785ae42a3..ff72493971b 100644 --- a/tests/cfgs/subclassification_disable/result/anydesk.pcapng.out +++ b/tests/cfgs/subclassification_disable/result/anydesk.pcapng.out @@ -10,7 +10,7 @@ LRU cache mining: 0/0/0 (insert/search/found) LRU cache msteams: 0/0/0 (insert/search/found) LRU cache fpc_dns: 2/4/0 (insert/search/found) Automa host: 4/4 (search/found) -Automa domain: 0/0 (search/found) +Automa domain: 4/0 (search/found) Automa tls cert: 0/0 (search/found) Automa risk mask: 2/0 (search/found) Automa common alpns: 1/0 (search/found) diff --git a/tests/cfgs/subclassification_disable/result/dns.pcap.out b/tests/cfgs/subclassification_disable/result/dns.pcap.out index 6046db33211..2e62be9090b 100644 --- a/tests/cfgs/subclassification_disable/result/dns.pcap.out +++ b/tests/cfgs/subclassification_disable/result/dns.pcap.out @@ -9,7 +9,7 @@ LRU cache mining: 0/0/0 (insert/search/found) LRU cache msteams: 0/0/0 (insert/search/found) LRU cache fpc_dns: 1/0/0 (insert/search/found) Automa host: 3/3 (search/found) -Automa domain: 0/0 (search/found) +Automa domain: 3/0 (search/found) Automa tls cert: 0/0 (search/found) Automa risk mask: 2/0 (search/found) Automa common alpns: 0/0 (search/found)