From c5d032afec66314f8e040aa75bdb8a8d87bfb85b Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Mon, 4 Nov 2024 10:25:31 +0100 Subject: [PATCH 1/7] ndpi: initial implementation of nDPI plugin Ticket: #7231 --- configure.ac | 55 +++ doc/userguide/rules/index.rst | 2 + doc/userguide/rules/ndpi-protocol.rst | 43 +++ doc/userguide/rules/ndpi-risk.rst | 49 +++ plugins/Makefile.am | 4 + plugins/ndpi/Makefile.am | 13 + plugins/ndpi/ndpi.c | 535 ++++++++++++++++++++++++++ suricata.yaml.in | 1 + 8 files changed, 702 insertions(+) create mode 100644 doc/userguide/rules/ndpi-protocol.rst create mode 100644 doc/userguide/rules/ndpi-risk.rst create mode 100644 plugins/ndpi/Makefile.am create mode 100644 plugins/ndpi/ndpi.c diff --git a/configure.ac b/configure.ac index ca964d9039a0..bdd8baedc02d 100644 --- a/configure.ac +++ b/configure.ac @@ -2309,6 +2309,57 @@ fi ]) AC_SUBST(RUST_FEATURES) +# nDPI support (no library checks for this stub) + NDPI_HOME= + AC_ARG_ENABLE(ndpi, + AS_HELP_STRING([--enable-ndpi], [Enable nDPI support]), + [enable_ndpi=$enableval],[enable_ndpi=no]) + AC_ARG_WITH([ndpi], + [ --with-ndpi= path to nDPI source tree.], + [NDPI_HOME="$withval"]) + + # Require --with-ndpi to be provided with an argument. + AS_IF([test "x$NDPI_HOME" = "xyes"], [ + AC_MSG_ERROR([--with-ndpi requires a path]) + exit 1 + ]) + + AS_IF([test "x$enable_dpi" = "xyes"], [ + if test "x$enable_shared" = "xno"; then + echo + echo " ERROR! ndpi cannot be enabled with --disable-shared" + echo + exit 1 + fi + ]) + + if test "x$enable_ndpi" = "xyes"; then + AC_MSG_CHECKING(for nDPI source) + if test "x$NDPI_HOME" != "x"; then + AC_MSG_RESULT(found in $NDPI_HOME) + NDPI_LIB=$NDPI_HOME/src/lib/libndpi.a + AC_MSG_CHECKING(for $NDPI_LIB) + if test -r $NDPI_LIB ; then : + AC_MSG_RESULT(found $NDPI_LIB) + fi + CPPFLAGS="${CPPFLAGS} -I$NDPI_HOME/src/include" + NDPI_LIB="$NDPI_HOME/src/lib/libndpi.a" + AC_SUBST([NDPI_LIB]) + else + AC_MSG_RESULT(not found) + enable_ndpi="no" + fi + fi + + if test "x$enable_ndpi" = "xyes"; then + AM_CONDITIONAL([BUILD_NDPI], [true]) + ndpi_comment="" + else + AM_CONDITIONAL([BUILD_NDPI], [false]) + ndpi_comment="#" + fi + AC_SUBST([ndpi_comment]) + AC_ARG_ENABLE(warnings, AS_HELP_STRING([--enable-warnings], [Enable supported C compiler warnings]),[enable_warnings=$enableval],[enable_warnings=no]) AS_IF([test "x$enable_warnings" = "xyes"], [ @@ -2530,6 +2581,7 @@ AC_CONFIG_FILES(examples/lib/simple/Makefile examples/lib/simple/Makefile.exampl AC_CONFIG_FILES(plugins/Makefile) AC_CONFIG_FILES(plugins/pfring/Makefile) AC_CONFIG_FILES(plugins/napatech/Makefile) +AC_CONFIG_FILES(plugins/ndpi/Makefile) AC_OUTPUT @@ -2585,6 +2637,9 @@ SURICATA_BUILD_CONF="Suricata Configuration: Plugin support (experimental): ${plugin_support} DPDK Bond PMD: ${enable_dpdk_bond_pmd} +Plugins: + nDPI: ${enable_ndpi} + Development settings: Coccinelle / spatch: ${enable_coccinelle} Unit tests enabled: ${enable_unittests} diff --git a/doc/userguide/rules/index.rst b/doc/userguide/rules/index.rst index efe3d83137d8..2846cf36916b 100644 --- a/doc/userguide/rules/index.rst +++ b/doc/userguide/rules/index.rst @@ -38,6 +38,8 @@ Suricata Rules smtp-keywords websocket-keywords app-layer + ndpi-protocol + ndpi-risk xbits noalert thresholding diff --git a/doc/userguide/rules/ndpi-protocol.rst b/doc/userguide/rules/ndpi-protocol.rst new file mode 100644 index 000000000000..f6ea4a6a0439 --- /dev/null +++ b/doc/userguide/rules/ndpi-protocol.rst @@ -0,0 +1,43 @@ +nDPI Protocol Keyword +===================== + +ndpi-protocol +------------- + +Match on the Layer-7 protocol detected by nDPI. + +Suricata should be compiled with the nDPI support and the ``ndpi`` +plugin must be loaded before it can be used. + +Example of configuring Suricata to be compiled with nDPI support: + +.. code-block:: console + + ./configure --enable-ndpi --with-ndpi=/home/user/nDPI + +Example of suricata.yaml configuration file to load the ``ndpi`` plugin:: + + plugins: + - /usr/lib/suricata/ndpi.so + +Syntax:: + + ndpi-protocol:[!]; + +Where protocol is one of the application protocols detected by nDPI. +Plase check ndpiReader -H for the full list. +It is possible to specify the transport protocol, the application +protocol, or both (dot-separated). + +Examples:: + + ndpi-protocol:HTTP; + ndpi-protocol:!TLS; + ndpi-protocol:TLS.YouTube; + +Here is an example of a rule matching TLS traffic on port 53: + +.. container:: example-rule + + alert tcp any any -> any 53 (msg:"TLS traffic over DNS standard port"; ndpi-protocol:TLS; sid:1;) + diff --git a/doc/userguide/rules/ndpi-risk.rst b/doc/userguide/rules/ndpi-risk.rst new file mode 100644 index 000000000000..41b36b700d00 --- /dev/null +++ b/doc/userguide/rules/ndpi-risk.rst @@ -0,0 +1,49 @@ +nDPI Risk Keyword +================= + +ndpi-risk +--------- + +Match on the flow risks detected by nDPI. Risks are potential issues detected +by nDPI during the packet dissection and include: + +- Known Proto on Non Std Port +- Binary App Transfer +- Self-signed Certificate +- Susp DGA Domain name +- Malware host contacted +- and many other... + +Suricata should be compiled with the nDPI support and the ``ndpi`` +plugin must be loaded before it can be used. + +Example of configuring Suricata to be compiled with nDPI support: + +.. code-block:: console + + ./configure --enable-ndpi --with-ndpi=/home/user/nDPI + +Example of suricata.yaml configuration file to load the ``ndpi`` plugin:: + + plugins: + - /usr/lib/suricata/ndpi.so + +Syntax:: + + ndpi-risk:[!]; + +Where risk is one (or multiple comma-separated) of the risk codes supported by +nDPI (e.g. NDPI_BINARY_APPLICATION_TRANSFER). Please check ndpiReader -H for the +full list. + +Examples:: + + ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; + ndpi-risk:NDPI_TLS_OBSOLETE_VERSION,NDPI_TLS_WEAK_CIPHER; + +Here is an example of a rule matching HTTP traffic transferring a binary application: + +.. container:: example-rule + + alert tcp any any -> any any (msg:"Binary application transfer over HTTP"; ndpi-protocol:HTTP; ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; sid:1;) + diff --git a/plugins/Makefile.am b/plugins/Makefile.am index cb7041326031..5d2dca1f5bc1 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -7,3 +7,7 @@ endif if BUILD_NAPATECH SUBDIRS += napatech endif + +if BUILD_NDPI +SUBDIRS += ndpi +endif diff --git a/plugins/ndpi/Makefile.am b/plugins/ndpi/Makefile.am new file mode 100644 index 000000000000..4d5dcbe4ed0d --- /dev/null +++ b/plugins/ndpi/Makefile.am @@ -0,0 +1,13 @@ +pkglib_LTLIBRARIES = ndpi.la + +ndpi_la_LDFLAGS = -module -avoid-version -shared +ndpi_la_LIBADD = @NDPI_LIB@ + +# Only required to find these headers when building plugins from the +# source directory. +ndpi_la_CFLAGS = -I../../rust/gen -I../../rust/dist + +ndpi_la_SOURCES = ndpi.c + +install-exec-hook: + cd $(DESTDIR)$(pkglibdir) && $(RM) $(pkglib_LTLIBRARIES) diff --git a/plugins/ndpi/ndpi.c b/plugins/ndpi/ndpi.c new file mode 100644 index 000000000000..1723c0754b85 --- /dev/null +++ b/plugins/ndpi/ndpi.c @@ -0,0 +1,535 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/* License note: While this "glue" code to the nDPI library is GPLv2, + * nDPI is itself LGPLv3 which is known to be incompatible with the + * GPLv2. */ + +#include "suricata-common.h" +#include "suricata-plugin.h" + +#include "detect-engine-helper.h" +#include "detect-parse.h" +#include "flow-callbacks.h" +#include "flow-storage.h" +#include "output-eve.h" +#include "thread-callbacks.h" +#include "thread-storage.h" +#include "util-debug.h" + +#include "ndpi_api.h" + +static ThreadStorageId thread_storage_id = { .id = -1 }; +static FlowStorageId flow_storage_id = { .id = -1 }; +static int ndpi_protocol_keyword_id = -1; +static int ndpi_risk_keyword_id = -1; + +struct NdpiThreadContext { + struct ndpi_detection_module_struct *ndpi; +}; + +struct NdpiFlowContext { + struct ndpi_flow_struct *ndpi_flow; + ndpi_protocol detected_l7_protocol; + bool detection_completed; +}; + +typedef struct DetectnDPIProtocolData_ { + ndpi_master_app_protocol l7_protocol; + bool negated; +} DetectnDPIProtocolData; + +typedef struct DetectnDPIRiskData_ { + ndpi_risk risk_mask; /* uint64 */ + bool negated; +} DetectnDPIRiskData; + +static void ThreadStorageFree(void *ptr) +{ + SCLogDebug("Free'ing nDPI thread storage"); + struct NdpiThreadContext *context = ptr; + ndpi_exit_detection_module(context->ndpi); + SCFree(context); +} + +static void FlowStorageFree(void *ptr) +{ + struct NdpiFlowContext *ctx = ptr; + ndpi_flow_free(ctx->ndpi_flow); + SCFree(ctx); +} + +static void OnFlowInit(ThreadVars *tv, Flow *f, const Packet *p, void *_data) +{ + struct NdpiFlowContext *flowctx = SCCalloc(1, sizeof(*flowctx)); + if (flowctx == NULL) { + FatalError("Failed to allocate nDPI flow context"); + } + + flowctx->ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT); + if (flowctx->ndpi_flow == NULL) { + FatalError("Failed to allocate nDPI flow"); + } + + memset(flowctx->ndpi_flow, 0, SIZEOF_FLOW_STRUCT); + flowctx->detection_completed = false; + FlowSetStorageById(f, flow_storage_id, flowctx); +} + +static void OnFlowUpdate(ThreadVars *tv, Flow *f, Packet *p, void *_data) +{ + struct NdpiThreadContext *threadctx = ThreadGetStorageById(tv, thread_storage_id); + struct NdpiFlowContext *flowctx = FlowGetStorageById(f, flow_storage_id); + uint16_t ip_len = 0; + void *ip_ptr = NULL; + + if (!threadctx->ndpi || !flowctx->ndpi_flow) { + return; + } + + if (PacketIsIPv4(p)) { + const IPV4Hdr *ip4h = PacketGetIPv4(p); + ip_len = IPV4_GET_RAW_IPLEN(ip4h); + ip_ptr = (void *)PacketGetIPv4(p); + } else if (PacketIsIPv6(p)) { + const IPV6Hdr *ip6h = PacketGetIPv6(p); + ip_len = IPV6_HEADER_LEN + IPV6_GET_RAW_PLEN(ip6h); + ip_ptr = (void *)PacketGetIPv6(p); + } + + if (!flowctx->detection_completed && ip_ptr != NULL && ip_len > 0) { + uint64_t time_ms = ((uint64_t)p->ts.secs) * 1000 + p->ts.usecs / 1000; + + SCLogDebug("Performing nDPI detection..."); + + flowctx->detected_l7_protocol = ndpi_detection_process_packet( + threadctx->ndpi, flowctx->ndpi_flow, ip_ptr, ip_len, time_ms, NULL); + + if (ndpi_is_protocol_detected(flowctx->detected_l7_protocol) != 0) { + if (!ndpi_is_proto_unknown(flowctx->detected_l7_protocol.proto)) { + if (!ndpi_extra_dissection_possible(threadctx->ndpi, flowctx->ndpi_flow)) + flowctx->detection_completed = true; + } + } else { + uint16_t max_num_pkts = (f->proto == IPPROTO_UDP) ? 8 : 24; + + if ((f->todstpktcnt + f->tosrcpktcnt) > max_num_pkts) { + uint8_t proto_guessed; + + flowctx->detected_l7_protocol = + ndpi_detection_giveup(threadctx->ndpi, flowctx->ndpi_flow, &proto_guessed); + flowctx->detection_completed = true; + } + } + + if (flowctx->detection_completed) { + SCLogDebug("Detected protocol: %s | app protocol: %s | category: %s", + ndpi_get_proto_name( + threadctx->ndpi, flowctx->detected_l7_protocol.proto.master_protocol), + ndpi_get_proto_name( + threadctx->ndpi, flowctx->detected_l7_protocol.proto.app_protocol), + ndpi_category_get_name( + threadctx->ndpi, flowctx->detected_l7_protocol.category)); + } + } +} + +static void OnFlowFinish(ThreadVars *tv, Flow *f, void *_data) +{ + /* Nothing to do here, the storage API has taken care of cleaning + * up storage, just here for example purposes. */ + SCLogDebug("Flow %p is now finished", f); +} + +static void OnThreadInit(ThreadVars *tv, void *_data) +{ + struct NdpiThreadContext *context = SCCalloc(1, sizeof(*context)); + if (context == NULL) { + FatalError("Failed to allocate nDPI thread context"); + } + context->ndpi = ndpi_init_detection_module(NULL); + if (context->ndpi == NULL) { + FatalError("Failed to initialize nDPI detection module"); + } + NDPI_PROTOCOL_BITMASK protos; + NDPI_BITMASK_SET_ALL(protos); + ndpi_set_protocol_detection_bitmask2(context->ndpi, &protos); + ndpi_finalize_initialization(context->ndpi); + ThreadSetStorageById(tv, thread_storage_id, context); +} + +static int DetectnDPIProtocolPacketMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + const Flow *f = p->flow; + struct NdpiFlowContext *flowctx = FlowGetStorageById(f, flow_storage_id); + const DetectnDPIProtocolData *data = (const DetectnDPIProtocolData *)ctx; + + SCEnter(); + + /* if the sig is PD-only we only match when PD packet flags are set */ + /* + if (s->type == SIG_TYPE_PDONLY && + (p->flags & (PKT_PROTO_DETECT_TS_DONE | PKT_PROTO_DETECT_TC_DONE)) == 0) { + SCLogDebug("packet %"PRIu64": flags not set", p->pcap_cnt); + SCReturnInt(0); + } + */ + + if (!flowctx->detection_completed) { + SCLogDebug("packet %" PRIu64 ": ndpi protocol not yet detected", p->pcap_cnt); + SCReturnInt(0); + } + + if (f == NULL) { + SCLogDebug("packet %" PRIu64 ": no flow", p->pcap_cnt); + SCReturnInt(0); + } + + bool r = ndpi_is_proto_equals(flowctx->detected_l7_protocol.proto, data->l7_protocol, false); + r = r ^ data->negated; + + if (r) { + SCLogDebug("ndpi protocol match on protocol = %u.%u (match %u)", + flowctx->detected_l7_protocol.proto.app_protocol, + flowctx->detected_l7_protocol.proto.master_protocol, + data->l7_protocol.app_protocol); + SCReturnInt(1); + } + SCReturnInt(0); +} + +static DetectnDPIProtocolData *DetectnDPIProtocolParse(const char *arg, bool negate) +{ + DetectnDPIProtocolData *data; + struct ndpi_detection_module_struct *ndpi_struct; + ndpi_master_app_protocol l7_protocol; + char *l7_protocol_name = (char *)arg; + NDPI_PROTOCOL_BITMASK all; + + /* convert protocol name (string) to ID */ + ndpi_struct = ndpi_init_detection_module(NULL); + if (unlikely(ndpi_struct == NULL)) + return NULL; + + ndpi_struct = ndpi_init_detection_module(NULL); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + ndpi_finalize_initialization(ndpi_struct); + + l7_protocol = ndpi_get_protocol_by_name(ndpi_struct, l7_protocol_name); + ndpi_exit_detection_module(ndpi_struct); + + if (ndpi_is_proto_unknown(l7_protocol)) { + SCLogError("failure parsing nDPI protocol '%s'", l7_protocol_name); + return NULL; + } + + data = SCMalloc(sizeof(DetectnDPIProtocolData)); + if (unlikely(data == NULL)) + return NULL; + + memcpy(&data->l7_protocol, &l7_protocol, sizeof(ndpi_master_app_protocol)); + data->negated = negate; + + return data; +} + +static bool nDPIProtocolDataHasConflicts( + const DetectnDPIProtocolData *us, const DetectnDPIProtocolData *them) +{ + /* check for mix of negated and non negated */ + if (them->negated ^ us->negated) + return true; + + /* check for multiple non-negated */ + if (!us->negated) + return true; + + /* check for duplicate */ + if (ndpi_is_proto_equals(us->l7_protocol, them->l7_protocol, true)) + return true; + + return false; +} + +static int DetectnDPIProtocolSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) +{ + DetectnDPIProtocolData *data = DetectnDPIProtocolParse(arg, s->init_data->negated); + if (data == NULL) + goto error; + + SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for (; tsm != NULL; tsm = tsm->next) { + if (tsm->type == ndpi_protocol_keyword_id) { + const DetectnDPIProtocolData *them = (const DetectnDPIProtocolData *)tsm->ctx; + + if (nDPIProtocolDataHasConflicts(data, them)) { + SCLogError("can't mix " + "positive ndpi-protocol match with negated"); + goto error; + } + } + } + + if (SigMatchAppendSMToList(de_ctx, s, ndpi_protocol_keyword_id, (SigMatchCtx *)data, + DETECT_SM_LIST_MATCH) == NULL) { + goto error; + } + return 0; + +error: + if (data != NULL) + SCFree(data); + return -1; +} + +static void DetectnDPIProtocolFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +static int DetectnDPIRiskPacketMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + const Flow *f = p->flow; + struct NdpiFlowContext *flowctx = FlowGetStorageById(f, flow_storage_id); + const DetectnDPIRiskData *data = (const DetectnDPIRiskData *)ctx; + + SCEnter(); + + if (!flowctx->detection_completed) { + SCLogDebug("packet %" PRIu64 ": ndpi risks not yet detected", p->pcap_cnt); + SCReturnInt(0); + } + + if (f == NULL) { + SCLogDebug("packet %" PRIu64 ": no flow", p->pcap_cnt); + SCReturnInt(0); + } + + bool r = ((flowctx->ndpi_flow->risk & data->risk_mask) == data->risk_mask); + r = r ^ data->negated; + + if (r) { + SCLogDebug("ndpi risks match on risk bitmap = %" PRIu64 " (matching bitmap %" PRIu64 ")", + flowctx->ndpi_flow->risk, data->risk_mask); + SCReturnInt(1); + } + + SCReturnInt(0); +} + +static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) +{ + DetectnDPIRiskData *data; + struct ndpi_detection_module_struct *ndpi_struct; + ndpi_risk risk_mask; + NDPI_PROTOCOL_BITMASK all; + + /* convert list of risk names (string) to mask */ + ndpi_struct = ndpi_init_detection_module(NULL); + if (unlikely(ndpi_struct == NULL)) + return NULL; + + ndpi_struct = ndpi_init_detection_module(NULL); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + ndpi_finalize_initialization(ndpi_struct); + + if (isdigit(arg[0])) + risk_mask = atoll(arg); + else { + char *dup = SCStrdup(arg), *tmp, *token; + + NDPI_ZERO_BIT(risk_mask); + + if (dup != NULL) { + token = strtok_r(dup, ",", &tmp); + + while (token != NULL) { + ndpi_risk_enum risk_id = ndpi_code2risk(token); + if (risk_id >= NDPI_MAX_RISK) { + SCLogError("unrecognized risk '%s', " + "please check ndpiReader -H for valid risk codes", + token); + return NULL; + } + NDPI_SET_BIT(risk_mask, risk_id); + token = strtok_r(NULL, ",", &tmp); + } + + SCFree(dup); + } + } + + data = SCMalloc(sizeof(DetectnDPIRiskData)); + if (unlikely(data == NULL)) + return NULL; + + data->risk_mask = risk_mask; + data->negated = negate; + + return data; +} + +static bool nDPIRiskDataHasConflicts(const DetectnDPIRiskData *us, const DetectnDPIRiskData *them) +{ + /* check for duplicate */ + if (us->risk_mask == them->risk_mask) + return true; + + return false; +} + +static int DetectnDPIRiskSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) +{ + DetectnDPIRiskData *data = DetectnDPIRiskParse(arg, s->init_data->negated); + if (data == NULL) + goto error; + + SigMatch *tsm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for (; tsm != NULL; tsm = tsm->next) { + if (tsm->type == ndpi_risk_keyword_id) { + const DetectnDPIRiskData *them = (const DetectnDPIRiskData *)tsm->ctx; + + if (nDPIRiskDataHasConflicts(data, them)) { + SCLogError("can't mix " + "positive ndpi-risk match with negated"); + goto error; + } + } + } + + if (SigMatchAppendSMToList(de_ctx, s, ndpi_risk_keyword_id, (SigMatchCtx *)data, + DETECT_SM_LIST_MATCH) == NULL) { + goto error; + } + return 0; + +error: + if (data != NULL) + SCFree(data); + return -1; +} + +static void DetectnDPIRiskFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +static void EveCallback(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb, void *data) +{ + struct NdpiThreadContext *threadctx = ThreadGetStorageById(tv, thread_storage_id); + struct NdpiFlowContext *flowctx = FlowGetStorageById(f, flow_storage_id); + ndpi_serializer serializer; + char *buffer; + uint32_t buffer_len; + + SCLogDebug("EveCallback: tv=%p, p=%p, f=%p", tv, p, f); + + if (f == NULL) + return; + + ndpi_init_serializer(&serializer, ndpi_serialization_format_inner_json); + + /* Use ndpi_dpi2json to get a JSON with nDPI metadata */ + ndpi_dpi2json(threadctx->ndpi, flowctx->ndpi_flow, flowctx->detected_l7_protocol, &serializer); + + buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len); + + /* Inject the nDPI JSON to the JsonBuilder */ + jb_set_formatted(jb, buffer); + + ndpi_term_serializer(&serializer); +} + +static void NdpInitRiskKeyword(void) +{ + /* SCSigTableElmt and DetectHelperKeywordRegister don't yet + * support all the fields required to register the nDPI keywords, + * so we'll just register with an empty keyword specifier to get + * the ID, then fill in the ID. */ + SCSigTableElmt keyword = {}; + ndpi_protocol_keyword_id = DetectHelperKeywordRegister(&keyword); + SCLogDebug("Registered new ndpi-protocol keyword with ID %" PRIu32, ndpi_protocol_keyword_id); + + sigmatch_table[ndpi_protocol_keyword_id].name = "ndpi-protocol"; + sigmatch_table[ndpi_protocol_keyword_id].desc = "match on the detected nDPI protocol"; + sigmatch_table[ndpi_protocol_keyword_id].url = "/rules/ndpi-protocol.html"; + sigmatch_table[ndpi_protocol_keyword_id].Match = DetectnDPIProtocolPacketMatch; + sigmatch_table[ndpi_protocol_keyword_id].Setup = DetectnDPIProtocolSetup; + sigmatch_table[ndpi_protocol_keyword_id].Free = DetectnDPIProtocolFree; + sigmatch_table[ndpi_protocol_keyword_id].flags = + (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); + + ndpi_risk_keyword_id = DetectHelperKeywordRegister(&keyword); + SCLogDebug("Registered new ndpi-risk keyword with ID %" PRIu32, ndpi_risk_keyword_id); + + sigmatch_table[ndpi_risk_keyword_id].name = "ndpi-risk"; + sigmatch_table[ndpi_risk_keyword_id].desc = "match on the detected nDPI risk"; + sigmatch_table[ndpi_risk_keyword_id].url = "/rules/ndpi-risk.html"; + sigmatch_table[ndpi_risk_keyword_id].Match = DetectnDPIRiskPacketMatch; + sigmatch_table[ndpi_risk_keyword_id].Setup = DetectnDPIRiskSetup; + sigmatch_table[ndpi_risk_keyword_id].Free = DetectnDPIRiskFree; + sigmatch_table[ndpi_risk_keyword_id].flags = + (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); +} + +static void NdpiInit(void) +{ + SCLogDebug("Initializing nDPI plugin"); + + /* Register thread storage. */ + thread_storage_id = ThreadStorageRegister("ndpi", sizeof(void *), NULL, ThreadStorageFree); + if (thread_storage_id.id < 0) { + FatalError("Failed to register nDPI thread storage"); + } + + /* Register flow storage. */ + flow_storage_id = FlowStorageRegister("ndpi", sizeof(void *), NULL, FlowStorageFree); + if (flow_storage_id.id < 0) { + FatalError("Failed to register nDPI flow storage"); + } + + /* Register flow lifecycle callbacks. */ + SCFlowRegisterInitCallback(OnFlowInit, NULL); + SCFlowRegisterUpdateCallback(OnFlowUpdate, NULL); + + /* Not needed for nDPI, but exists for completeness. */ + SCFlowRegisterFinishCallback(OnFlowFinish, NULL); + + /* Register thread init callback. */ + SCThreadRegisterInitCallback(OnThreadInit, NULL); + + /* Register an EVE callback. */ + SCEveRegisterCallback(EveCallback, NULL); + + NdpInitRiskKeyword(); +} + +const SCPlugin PluginRegistration = { + .name = "ndpi", + .author = "Luca Deri", + .license = "GPLv3", + .Init = NdpiInit, +}; + +const SCPlugin *SCPluginRegister() +{ + return &PluginRegistration; +} diff --git a/suricata.yaml.in b/suricata.yaml.in index 4bc9e87aa2af..e16360a4699d 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -82,6 +82,7 @@ stats: plugins: @pfring_comment@- @prefix@/lib/@PACKAGE_NAME@/pfring.so @napatech_comment@- @prefix@/lib/@PACKAGE_NAME@/napatech.so + @ndpi_comment@- @prefix@/lib/@PACKAGE_NAME@/ndpi.so # - /path/to/plugin.so # Configure the type of alert (and other) logging you would like. From 2b16f3ace762a2b71f94148a769d61cd2c81b802 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 15 Jan 2025 12:04:02 -0600 Subject: [PATCH 2/7] github-ci: add ndpi build to the centos-stream9 build - Download and build nDPI - Enable nDPI during Suricata ./configure - Test that the plugin was built and installed --- .github/workflows/builds.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 04a2f1d5102c..0deba92e8c5c 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -628,9 +628,19 @@ jobs: uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 with: name: dist + + - name: Build and install nDPI + run: | + curl -OL https://github.com/ntop/nDPI/archive/refs/tags/4.12.tar.gz + tar xvf 4.12.tar.gz + cd nDPI-4.12 + ./autogen.sh + ./configure + make -j ${{ env.CPUS }} + - run: tar zxvf suricata-*.tar.gz --strip-components=1 - name: ./configure - run: CFLAGS="${DEFAULT_CFLAGS}" ./configure + run: CFLAGS="${DEFAULT_CFLAGS}" ./configure --enable-ndpi --with-ndpi=$(pwd)/nDPI-4.12 - run: make -j ${{ env.CPUS }} - run: make install - run: make install-conf @@ -648,6 +658,8 @@ jobs: with: name: prep path: prep + - name: Check if the nDPI plugin was installed + run: test -e /usr/local/lib/suricata/ndpi.so - run: tar xf prep/suricata-verify.tar.gz - run: python3 ./suricata-verify/run.py -q --debug-failed - run: suricata-update -V From 0f6a936d778dc6b3b0d4cb1f5d61b27495e09ba8 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 15 Jan 2025 15:25:20 -0600 Subject: [PATCH 3/7] eve/schema: add top level ndpi object The format is left free-form, as its controled by a plugin. --- etc/schema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/schema.json b/etc/schema.json index 818ad8a24dc8..4cf25dde1526 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -6866,6 +6866,10 @@ } }, "additionalProperties": false + }, + "ndpi": { + "description": "nDPI plugin", + "type": "object" } }, "$defs": { From 120f08201d3e939c52adc028dbeaff2204123b99 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 16 Jan 2025 14:10:39 -0600 Subject: [PATCH 4/7] ndpi: check for flow earlier in eve callback The eve callback in ndpi requires a flow, so bail earlier if there is no flow. --- plugins/ndpi/ndpi.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/ndpi/ndpi.c b/plugins/ndpi/ndpi.c index 1723c0754b85..c2d4c823c3ad 100644 --- a/plugins/ndpi/ndpi.c +++ b/plugins/ndpi/ndpi.c @@ -434,6 +434,11 @@ static void DetectnDPIRiskFree(DetectEngineCtx *de_ctx, void *ptr) static void EveCallback(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb, void *data) { + /* Adding ndpi info to EVE requires a flow. */ + if (f == NULL) { + return; + } + struct NdpiThreadContext *threadctx = ThreadGetStorageById(tv, thread_storage_id); struct NdpiFlowContext *flowctx = FlowGetStorageById(f, flow_storage_id); ndpi_serializer serializer; @@ -442,9 +447,6 @@ static void EveCallback(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *j SCLogDebug("EveCallback: tv=%p, p=%p, f=%p", tv, p, f); - if (f == NULL) - return; - ndpi_init_serializer(&serializer, ndpi_serialization_format_inner_json); /* Use ndpi_dpi2json to get a JSON with nDPI metadata */ From 9954e184df1151d7fab58e9ff74a6b69f009d539 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 17 Jan 2025 14:46:07 -0600 Subject: [PATCH 5/7] detect: split new keyword id from registration Split DetectHelperKeywordRegister into 2 functions, one for acquiring a new keyword ID, and another to perform the registration. This makes it easier to do the traditional C keyword initialization with a dynamic ID. --- plugins/ndpi/ndpi.c | 5 ++- src/detect-engine-helper.c | 62 +++++++++++++++++++++----------------- src/detect-engine-helper.h | 2 ++ 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/plugins/ndpi/ndpi.c b/plugins/ndpi/ndpi.c index c2d4c823c3ad..544a1cad13c0 100644 --- a/plugins/ndpi/ndpi.c +++ b/plugins/ndpi/ndpi.c @@ -466,8 +466,7 @@ static void NdpInitRiskKeyword(void) * support all the fields required to register the nDPI keywords, * so we'll just register with an empty keyword specifier to get * the ID, then fill in the ID. */ - SCSigTableElmt keyword = {}; - ndpi_protocol_keyword_id = DetectHelperKeywordRegister(&keyword); + ndpi_protocol_keyword_id = SCDetectHelperNewKeywordId(); SCLogDebug("Registered new ndpi-protocol keyword with ID %" PRIu32, ndpi_protocol_keyword_id); sigmatch_table[ndpi_protocol_keyword_id].name = "ndpi-protocol"; @@ -479,7 +478,7 @@ static void NdpInitRiskKeyword(void) sigmatch_table[ndpi_protocol_keyword_id].flags = (SIGMATCH_QUOTES_OPTIONAL | SIGMATCH_HANDLE_NEGATION); - ndpi_risk_keyword_id = DetectHelperKeywordRegister(&keyword); + ndpi_risk_keyword_id = SCDetectHelperNewKeywordId(); SCLogDebug("Registered new ndpi-risk keyword with ID %" PRIu32, ndpi_risk_keyword_id); sigmatch_table[ndpi_risk_keyword_id].name = "ndpi-risk"; diff --git a/src/detect-engine-helper.c b/src/detect-engine-helper.c index 07ffb8177057..15290d0a52f0 100644 --- a/src/detect-engine-helper.c +++ b/src/detect-engine-helper.c @@ -95,7 +95,7 @@ int DetectHelperMultiBufferMpmRegister(const char *name, const char *desc, AppPr return DetectBufferTypeGetByName(name); } -int DetectHelperKeywordRegister(const SCSigTableElmt *kw) +int SCDetectHelperNewKeywordId(void) { if (DETECT_TBLSIZE_IDX >= DETECT_TBLSIZE) { void *tmp = SCRealloc( @@ -107,45 +107,51 @@ int DetectHelperKeywordRegister(const SCSigTableElmt *kw) DETECT_TBLSIZE += DETECT_TBLSIZE_STEP; } - sigmatch_table[DETECT_TBLSIZE_IDX].name = kw->name; - sigmatch_table[DETECT_TBLSIZE_IDX].desc = kw->desc; - sigmatch_table[DETECT_TBLSIZE_IDX].url = kw->url; - sigmatch_table[DETECT_TBLSIZE_IDX].flags = kw->flags; - sigmatch_table[DETECT_TBLSIZE_IDX].AppLayerTxMatch = + DETECT_TBLSIZE_IDX++; + return DETECT_TBLSIZE_IDX - 1; +} + +int DetectHelperKeywordRegister(const SCSigTableElmt *kw) +{ + int keyword_id = SCDetectHelperNewKeywordId(); + if (keyword_id < 0) { + return -1; + } + + sigmatch_table[keyword_id].name = kw->name; + sigmatch_table[keyword_id].desc = kw->desc; + sigmatch_table[keyword_id].url = kw->url; + sigmatch_table[keyword_id].flags = kw->flags; + sigmatch_table[keyword_id].AppLayerTxMatch = (int (*)(DetectEngineThreadCtx * det_ctx, Flow * f, uint8_t flags, void *alstate, void *txv, const Signature *s, const SigMatchCtx *ctx)) kw->AppLayerTxMatch; - sigmatch_table[DETECT_TBLSIZE_IDX].Setup = + sigmatch_table[keyword_id].Setup = (int (*)(DetectEngineCtx * de, Signature * s, const char *raw)) kw->Setup; - sigmatch_table[DETECT_TBLSIZE_IDX].Free = (void (*)(DetectEngineCtx * de, void *ptr)) kw->Free; - DETECT_TBLSIZE_IDX++; - return DETECT_TBLSIZE_IDX - 1; + sigmatch_table[keyword_id].Free = (void (*)(DetectEngineCtx * de, void *ptr)) kw->Free; + + return keyword_id; } int DetectHelperTransformRegister(const SCTransformTableElmt *kw) { - if (DETECT_TBLSIZE_IDX >= DETECT_TBLSIZE) { - void *tmp = SCRealloc( - sigmatch_table, (DETECT_TBLSIZE + DETECT_TBLSIZE_STEP) * sizeof(SigTableElmt)); - if (unlikely(tmp == NULL)) { - return -1; - } - sigmatch_table = tmp; - DETECT_TBLSIZE += DETECT_TBLSIZE_STEP; + int transform_id = SCDetectHelperNewKeywordId(); + if (transform_id < 0) { + return -1; } - sigmatch_table[DETECT_TBLSIZE_IDX].name = kw->name; - sigmatch_table[DETECT_TBLSIZE_IDX].desc = kw->desc; - sigmatch_table[DETECT_TBLSIZE_IDX].url = kw->url; - sigmatch_table[DETECT_TBLSIZE_IDX].flags = kw->flags; - sigmatch_table[DETECT_TBLSIZE_IDX].Transform = + sigmatch_table[transform_id].name = kw->name; + sigmatch_table[transform_id].desc = kw->desc; + sigmatch_table[transform_id].url = kw->url; + sigmatch_table[transform_id].flags = kw->flags; + sigmatch_table[transform_id].Transform = (void (*)(InspectionBuffer * buffer, void *options)) kw->Transform; - sigmatch_table[DETECT_TBLSIZE_IDX].TransformValidate = (bool (*)( + sigmatch_table[transform_id].TransformValidate = (bool (*)( const uint8_t *content, uint16_t content_len, void *context))kw->TransformValidate; - sigmatch_table[DETECT_TBLSIZE_IDX].Setup = + sigmatch_table[transform_id].Setup = (int (*)(DetectEngineCtx * de, Signature * s, const char *raw)) kw->Setup; - sigmatch_table[DETECT_TBLSIZE_IDX].Free = (void (*)(DetectEngineCtx * de, void *ptr)) kw->Free; - DETECT_TBLSIZE_IDX++; - return DETECT_TBLSIZE_IDX - 1; + sigmatch_table[transform_id].Free = (void (*)(DetectEngineCtx * de, void *ptr)) kw->Free; + + return transform_id; } InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ctx, diff --git a/src/detect-engine-helper.h b/src/detect-engine-helper.h index 39fa632ed28c..7c8044430850 100644 --- a/src/detect-engine-helper.h +++ b/src/detect-engine-helper.h @@ -28,6 +28,8 @@ #include "detect.h" #include "rust.h" +int SCDetectHelperNewKeywordId(void); + int DetectHelperKeywordRegister(const SCSigTableElmt *kw); int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver); From 32587049553029af6e22d6f89ec360c3b945b2c1 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 24 Jan 2025 10:51:45 -0600 Subject: [PATCH 6/7] doc/ndpi: add note about requires keyword Suggest that rules using ndpi keywords should also test for existence of the keyword with requires. --- doc/userguide/rules/ndpi-protocol.rst | 7 ++++++- doc/userguide/rules/ndpi-risk.rst | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/userguide/rules/ndpi-protocol.rst b/doc/userguide/rules/ndpi-protocol.rst index f6ea4a6a0439..7ae04d1ca257 100644 --- a/doc/userguide/rules/ndpi-protocol.rst +++ b/doc/userguide/rules/ndpi-protocol.rst @@ -15,6 +15,11 @@ Example of configuring Suricata to be compiled with nDPI support: ./configure --enable-ndpi --with-ndpi=/home/user/nDPI +Note that rules using the ``ndpi-protocol`` should check if the +``ndpi-protocol`` keyword exists with ``requires``, for example:: + + requires: keyword ndpi-protocol + Example of suricata.yaml configuration file to load the ``ndpi`` plugin:: plugins: @@ -39,5 +44,5 @@ Here is an example of a rule matching TLS traffic on port 53: .. container:: example-rule - alert tcp any any -> any 53 (msg:"TLS traffic over DNS standard port"; ndpi-protocol:TLS; sid:1;) + alert tcp any any -> any 53 (msg:"TLS traffic over DNS standard port"; requires: keyword ndpi-protocol; ndpi-protocol:TLS; sid:1;) diff --git a/doc/userguide/rules/ndpi-risk.rst b/doc/userguide/rules/ndpi-risk.rst index 41b36b700d00..2e43c3f207bb 100644 --- a/doc/userguide/rules/ndpi-risk.rst +++ b/doc/userguide/rules/ndpi-risk.rst @@ -14,8 +14,13 @@ by nDPI during the packet dissection and include: - Malware host contacted - and many other... -Suricata should be compiled with the nDPI support and the ``ndpi`` -plugin must be loaded before it can be used. +Suricata should be compiled with nDPI support and the ``ndpi`` plugin +must be loaded before it can be used. + +Additionally, rules using the ``ndpi-risk`` keyword should check if +the keyword exists using the ``requires`` keyword, for example:: + + ``requires: keyword ndpi-risk`` Example of configuring Suricata to be compiled with nDPI support: @@ -45,5 +50,5 @@ Here is an example of a rule matching HTTP traffic transferring a binary applica .. container:: example-rule - alert tcp any any -> any any (msg:"Binary application transfer over HTTP"; ndpi-protocol:HTTP; ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; sid:1;) + alert tcp any any -> any any (msg:"Binary application transfer over HTTP"; requires: keyword ndpi-protocol, keyword ndpi-risk; ndpi-protocol:HTTP; ndpi-risk:NDPI_BINARY_APPLICATION_TRANSFER; sid:1;) From 26611f6f1461165604c29aa659be3fe171f52c9e Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 24 Jan 2025 11:18:54 -0600 Subject: [PATCH 7/7] ndpi: fix memory in keyword setup - remove duplicate calls to ndpi_init_detection_module - cleanup ndpi_init_detection_module when no longer needed --- plugins/ndpi/ndpi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/ndpi/ndpi.c b/plugins/ndpi/ndpi.c index 544a1cad13c0..b1be986833ce 100644 --- a/plugins/ndpi/ndpi.c +++ b/plugins/ndpi/ndpi.c @@ -226,7 +226,6 @@ static DetectnDPIProtocolData *DetectnDPIProtocolParse(const char *arg, bool neg if (unlikely(ndpi_struct == NULL)) return NULL; - ndpi_struct = ndpi_init_detection_module(NULL); NDPI_BITMASK_SET_ALL(all); ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); ndpi_finalize_initialization(ndpi_struct); @@ -346,10 +345,10 @@ static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) if (unlikely(ndpi_struct == NULL)) return NULL; - ndpi_struct = ndpi_init_detection_module(NULL); NDPI_BITMASK_SET_ALL(all); ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); ndpi_finalize_initialization(ndpi_struct); + ndpi_exit_detection_module(ndpi_struct); if (isdigit(arg[0])) risk_mask = atoll(arg);