From 220894bc077269d04c582c07b935e677cbef0687 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 15 Feb 2025 19:20:14 +0100 Subject: [PATCH 01/17] detect/profile: convert rule grouping dump to json builder Ticket: #7558. --- src/detect-engine-build.c | 213 +++++++++++++++++++------------------- 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index 7c9b66013e35..fd73520ba9c1 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -599,7 +599,7 @@ static bool RuleMpmIsNegated(const Signature *s) return (cd->flags & DETECT_CONTENT_NEGATED) ? true : false; } -static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigGroupHead *sgh, +static JsonBuilder *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigGroupHead *sgh, const int add_rules, const int add_mpm_stats) { uint32_t prefilter_cnt = 0; @@ -636,14 +636,13 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG if (sgh->init == NULL) return NULL; - json_t *js = json_object(); + JsonBuilder *js = jb_new_object(); if (unlikely(js == NULL)) return NULL; - json_object_set_new(js, "id", json_integer(sgh->id)); - - json_t *js_array = json_array(); + jb_set_uint(js, "id", sgh->id); + jb_open_array(js, "rules"); for (uint32_t x = 0; x < sgh->init->sig_cnt; x++) { const Signature *s = sgh->init->match_array[x]; if (s == NULL) @@ -766,35 +765,37 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG alstats[s->alproto]++; if (add_rules) { - json_t *js_sig = json_object(); - if (unlikely(js == NULL)) - continue; - json_object_set_new(js_sig, "sig_id", json_integer(s->id)); - json_array_append_new(js_array, js_sig); + JsonBuilder *e = jb_new_object(); + if (e != NULL) { + jb_set_uint(e, "sig_id", s->id); + jb_close(e); + jb_append_object(js, e); + jb_free(e); + } } } + jb_close(js); - json_object_set_new(js, "rules", js_array); + jb_open_object(js, "stats"); + jb_set_uint(js, "total", sgh->init->sig_cnt); - json_t *stats = json_object(); - json_object_set_new(stats, "total", json_integer(sgh->init->sig_cnt)); - - json_t *types = json_object(); - json_object_set_new(types, "mpm", json_integer(mpm_cnt)); - json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt)); - json_object_set_new(types, "mpm_depth", json_integer(mpm_depth_cnt)); - json_object_set_new(types, "mpm_endswith", json_integer(mpm_endswith_cnt)); - json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt)); - json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt)); - json_object_set_new(types, "prefilter", json_integer(prefilter_cnt)); - json_object_set_new(types, "syn", json_integer(syn_cnt)); - json_object_set_new(types, "any5", json_integer(any5_cnt)); - json_object_set_new(stats, "types", types); + jb_open_object(js, "types"); + jb_set_uint(js, "mpm", mpm_cnt); + jb_set_uint(js, "non_mpm", nonmpm_cnt); + jb_set_uint(js, "mpm_depth", mpm_depth_cnt); + jb_set_uint(js, "mpm_endswith", mpm_endswith_cnt); + jb_set_uint(js, "negated_mpm", negmpm_cnt); + jb_set_uint(js, "payload_but_no_mpm", payload_no_mpm_cnt); + jb_set_uint(js, "prefilter", prefilter_cnt); + jb_set_uint(js, "syn", syn_cnt); + jb_set_uint(js, "any5", any5_cnt); + jb_close(js); for (AppProto i = 0; i < g_alproto_max; i++) { if (alstats[i] > 0) { - json_t *app = json_object(); - json_object_set_new(app, "total", json_integer(alstats[i])); + const char *proto_name = (i == ALPROTO_UNKNOWN) ? "payload" : AppProtoToString(i); + jb_open_object(js, proto_name); + jb_set_uint(js, "total", alstats[i]); for (int y = 0; y < max_buffer_type_id; y++) { if (alproto_mpm_bufs[i][y] == 0) @@ -806,54 +807,59 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG else name = DetectEngineBufferTypeGetNameById(de_ctx, y); - json_object_set_new(app, name, json_integer(alproto_mpm_bufs[i][y])); + jb_set_uint(js, name, alproto_mpm_bufs[i][y]); } - - const char *proto_name = (i == ALPROTO_UNKNOWN) ? "payload" : AppProtoToString(i); - json_object_set_new(stats, proto_name, app); + jb_close(js); } } if (add_mpm_stats) { - json_t *mpm_js = json_object(); + jb_open_object(js, "mpm"); for (int i = 0; i < max_buffer_type_id; i++) { if (mpm_stats[i].cnt > 0) { + const char *name; + if (i < DETECT_SM_LIST_DYNAMIC_START) + name = DetectListToHumanString(i); + else + name = DetectEngineBufferTypeGetNameById(de_ctx, i); + + jb_open_array(js, name); - json_t *mpm_sizes_array = json_array(); for (int y = 0; y < 256; y++) { if (mpm_sizes[i][y] == 0) continue; - json_t *e = json_object(); - json_object_set_new(e, "size", json_integer(y)); - json_object_set_new(e, "count", json_integer(mpm_sizes[i][y])); - json_array_append_new(mpm_sizes_array, e); + JsonBuilder *e = jb_new_object(); + if (e != NULL) { + jb_set_uint(e, "size", y); + jb_set_uint(e, "count", mpm_sizes[i][y]); + jb_close(e); + jb_append_object(js, e); + jb_free(e); + } } - json_t *buf = json_object(); - json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt)); - json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt)); - json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min)); - json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max)); - - json_object_set_new(buf, "sizes", mpm_sizes_array); - - const char *name; - if (i < DETECT_SM_LIST_DYNAMIC_START) - name = DetectListToHumanString(i); - else - name = DetectEngineBufferTypeGetNameById(de_ctx, i); + JsonBuilder *e = jb_new_object(); + if (e != NULL) { + jb_set_uint(e, "total", mpm_stats[i].cnt); + jb_set_uint(e, "avg_strength", mpm_stats[i].total / mpm_stats[i].cnt); + jb_set_uint(e, "min_strength", mpm_stats[i].min); + jb_set_uint(e, "max_strength", mpm_stats[i].max); + jb_close(e); + jb_append_object(js, e); + jb_free(e); + } - json_object_set_new(mpm_js, name, buf); + jb_close(js); } } - - json_object_set_new(stats, "mpm", mpm_js); + jb_close(js); } - json_object_set_new(js, "stats", stats); + jb_close(js); - json_object_set_new(js, "score", json_integer(sgh->init->score)); + jb_set_uint(js, "score", sgh->init->score); + jb_close(js); return js; } @@ -861,97 +867,90 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG static void RulesDumpGrouping(const DetectEngineCtx *de_ctx, const int add_rules, const int add_mpm_stats) { - json_t *js = json_object(); + JsonBuilder *js = jb_new_object(); if (unlikely(js == NULL)) return; - int p; - for (p = 0; p < 256; p++) { + for (int p = 0; p < 256; p++) { if (p == IPPROTO_TCP || p == IPPROTO_UDP) { const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp"; - json_t *tcp = json_object(); - - json_t *ts_array = json_array(); - DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp : - de_ctx->flow_gh[1].udp; + jb_open_object(js, name); + jb_open_array(js, "toserver"); + const DetectPort *list = + (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp : de_ctx->flow_gh[1].udp; while (list != NULL) { - json_t *port = json_object(); - json_object_set_new(port, "port", json_integer(list->port)); - json_object_set_new(port, "port2", json_integer(list->port2)); + JsonBuilder *port = jb_new_object(); + jb_set_uint(port, "port", list->port); + jb_set_uint(port, "port2", list->port2); - json_t *tcp_ts = + JsonBuilder *stats = RulesGroupPrintSghStats(de_ctx, list->sh, add_rules, add_mpm_stats); - json_object_set_new(port, "rulegroup", tcp_ts); - json_array_append_new(ts_array, port); + jb_set_object(port, "rulegroup", stats); + jb_free(stats); + jb_close(port); + jb_append_object(js, port); + jb_free(port); list = list->next; } - json_object_set_new(tcp, "toserver", ts_array); + jb_close(js); // toserver array - json_t *tc_array = json_array(); + jb_open_array(js, "toclient"); list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp : de_ctx->flow_gh[0].udp; while (list != NULL) { - json_t *port = json_object(); - json_object_set_new(port, "port", json_integer(list->port)); - json_object_set_new(port, "port2", json_integer(list->port2)); + JsonBuilder *port = jb_new_object(); + jb_set_uint(port, "port", list->port); + jb_set_uint(port, "port2", list->port2); - json_t *tcp_tc = + JsonBuilder *stats = RulesGroupPrintSghStats(de_ctx, list->sh, add_rules, add_mpm_stats); - json_object_set_new(port, "rulegroup", tcp_tc); - json_array_append_new(tc_array, port); + jb_set_object(port, "rulegroup", stats); + jb_free(stats); + jb_close(port); + jb_append_object(js, port); + jb_free(port); list = list->next; } - json_object_set_new(tcp, "toclient", tc_array); - - json_object_set_new(js, name, tcp); + jb_close(js); // toclient array + jb_close(js); } else if (p == IPPROTO_ICMP || p == IPPROTO_ICMPV6) { const char *name = (p == IPPROTO_ICMP) ? "icmpv4" : "icmpv6"; - json_t *o = json_object(); + jb_open_object(js, name); if (de_ctx->flow_gh[1].sgh[p]) { - json_t *ts = json_object(); - json_t *group_ts = RulesGroupPrintSghStats( + jb_open_object(js, "toserver"); + JsonBuilder *stats = RulesGroupPrintSghStats( de_ctx, de_ctx->flow_gh[1].sgh[p], add_rules, add_mpm_stats); - json_object_set_new(ts, "rulegroup", group_ts); - json_object_set_new(o, "toserver", ts); + jb_set_object(js, "rulegroup", stats); + jb_free(stats); + jb_close(js); } if (de_ctx->flow_gh[0].sgh[p]) { - json_t *tc = json_object(); - json_t *group_tc = RulesGroupPrintSghStats( + jb_open_object(js, "toclient"); + JsonBuilder *stats = RulesGroupPrintSghStats( de_ctx, de_ctx->flow_gh[0].sgh[p], add_rules, add_mpm_stats); - json_object_set_new(tc, "rulegroup", group_tc); - json_object_set_new(o, "toclient", tc); + jb_set_object(js, "rulegroup", stats); + jb_free(stats); + jb_close(js); } - json_object_set_new(js, name, o); + jb_close(js); } } + jb_close(js); const char *filename = "rule_group.json"; const char *log_dir = ConfigGetLogDirectory(); char log_path[PATH_MAX] = ""; - snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename); FILE *fp = fopen(log_path, "w"); - if (fp == NULL) { - return; - } - - char *js_s = json_dumps(js, - JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH); - if (unlikely(js_s == NULL)) { - fclose(fp); - return; + if (fp != NULL) { + fwrite(jb_ptr(js), jb_len(js), 1, fp); + (void)fclose(fp); } - - json_object_clear(js); - json_decref(js); - - fprintf(fp, "%s\n", js_s); - free(js_s); - fclose(fp); + jb_free(js); } static int RulesGroupByIPProto(DetectEngineCtx *de_ctx) From e938ad81e508cd3d17c077ecee6bece6dbc35db1 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 15 Feb 2025 11:23:44 +0100 Subject: [PATCH 02/17] smtp/events: set direction on rules Several rules matched on both directions even if events are set in a single direction. --- rules/smtp-events.rules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rules/smtp-events.rules b/rules/smtp-events.rules index 135b84c629e1..2898bdc299f4 100644 --- a/rules/smtp-events.rules +++ b/rules/smtp-events.rules @@ -8,12 +8,12 @@ # alert smtp any any -> any any (msg:"SURICATA SMTP invalid reply"; flow:established,to_client; app-layer-event:smtp.invalid_reply; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220000; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP unable to match reply with request"; flow:established,to_client; app-layer-event:smtp.unable_to_match_reply_with_request; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220001; rev:1;) -alert smtp any any -> any any (msg:"SURICATA SMTP max command line len exceeded"; flow:established; app-layer-event:smtp.max_command_line_len_exceeded; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220002; rev:1;) +alert smtp any any -> any any (msg:"SURICATA SMTP max command line len exceeded"; flow:established,to_server; app-layer-event:smtp.max_command_line_len_exceeded; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220002; rev:2;) alert smtp any any -> any any (msg:"SURICATA SMTP max reply line len exceeded"; flow:established,to_client; app-layer-event:smtp.max_reply_line_len_exceeded; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220003; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP invalid pipelined sequence"; flow:established,to_server; app-layer-event:smtp.invalid_pipelined_sequence; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220004; rev:1;) -alert smtp any any -> any any (msg:"SURICATA SMTP bdat chunk len exceeded"; flow:established; app-layer-event:smtp.bdat_chunk_len_exceeded; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220005; rev:1;) +alert smtp any any -> any any (msg:"SURICATA SMTP bdat chunk len exceeded"; flow:established,to_server; app-layer-event:smtp.bdat_chunk_len_exceeded; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220005; rev:2;) alert smtp any any -> any any (msg:"SURICATA SMTP no server welcome message"; flow:established,to_client; app-layer-event:smtp.no_server_welcome_message; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220006; rev:1;) -alert smtp any any -> any any (msg:"SURICATA SMTP tls rejected"; flow:established; app-layer-event:smtp.tls_rejected; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220007; rev:1;) +alert smtp any any -> any any (msg:"SURICATA SMTP tls rejected"; flow:established,to_client; app-layer-event:smtp.tls_rejected; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220007; rev:2;) alert smtp any any -> any any (msg:"SURICATA SMTP data command rejected"; flow:established,to_client; app-layer-event:smtp.data_command_rejected; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220008; rev:1;) # SMTP MIME events @@ -30,5 +30,5 @@ alert smtp any any -> any any (msg:"SURICATA SMTP Mime boundary length exceeded" alert smtp any any -> any any (msg:"SURICATA SMTP duplicate fields"; flow:established,to_server; app-layer-event:smtp.duplicate_fields; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220018; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP unparsable content"; flow:established,to_server; app-layer-event:smtp.unparsable_content; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220019; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP filename truncated"; flow:established,to_server; app-layer-event:smtp.mime_long_filename; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220020; rev:1;) -alert smtp any any -> any any (msg:"SURICATA SMTP failed protocol change"; flow:established; app-layer-event:smtp.failed_protocol_change; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220021; rev:1;) +alert smtp any any -> any any (msg:"SURICATA SMTP failed protocol change"; flow:established,to_client; app-layer-event:smtp.failed_protocol_change; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220021; rev:2;) # next sid 2220022 From d94ee5b84c7fe4669c8d5cdebb845b6df798006d Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 27 Jan 2025 10:55:46 +0100 Subject: [PATCH 03/17] detect: don't register duplicate app inspect engines --- src/detect-engine.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/detect-engine.c b/src/detect-engine.c index d576e550138a..d3d8077f8518 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -242,6 +242,21 @@ static void AppLayerInspectEngineRegisterInternal(const char *name, AppProto alp void DetectAppLayerInspectEngineRegister(const char *name, AppProto alproto, uint32_t dir, int progress, InspectEngineFuncPtr Callback, InspectionBufferGetDataPtr GetData) { + /* before adding, check that we don't add a duplicate entry, which will + * propegate all the way into the packet runtime if allowed. */ + DetectEngineAppInspectionEngine *t = g_app_inspect_engines; + while (t != NULL) { + const uint32_t t_direction = t->dir == 0 ? SIG_FLAG_TOSERVER : SIG_FLAG_TOCLIENT; + const int sm_list = DetectBufferTypeGetByName(name); + + if (t->sm_list == sm_list && t->alproto == alproto && t_direction == dir && + t->progress == progress && t->v2.Callback == Callback && t->v2.GetData == GetData) { + DEBUG_VALIDATE_BUG_ON(1); + return; + } + t = t->next; + } + AppLayerInspectEngineRegisterInternal(name, alproto, dir, progress, Callback, GetData, NULL); } From bc12994e24e926d93e71a9355e344f65be36f2cc Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 27 Jan 2025 11:17:34 +0100 Subject: [PATCH 04/17] detect/nfs: don't double register nfq_request generic list --- src/detect-nfs-version.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/detect-nfs-version.c b/src/detect-nfs-version.c index c24cb65629d5..d9c18075d94a 100644 --- a/src/detect-nfs-version.c +++ b/src/detect-nfs-version.c @@ -69,8 +69,6 @@ void DetectNfsVersionRegister (void) sigmatch_table[DETECT_NFS_VERSION].Setup = DetectNfsVersionSetup; sigmatch_table[DETECT_NFS_VERSION].Free = DetectNfsVersionFree; // unit tests were the same as DetectNfsProcedureRegisterTests - DetectAppLayerInspectEngineRegister( - "nfs_request", ALPROTO_NFS, SIG_FLAG_TOSERVER, 0, DetectEngineInspectGenericList, NULL); g_nfs_request_buffer_id = DetectBufferTypeGetByName("nfs_request"); From 187cd91ab3ea06f67b550581b06fa05d1b46d167 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 13 Feb 2025 10:32:32 +0100 Subject: [PATCH 05/17] detect/tls: don't double register tls_validity generic list --- src/detect-tls-cert-validity.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/detect-tls-cert-validity.c b/src/detect-tls-cert-validity.c index b8d9f184ecb4..9e858dc85112 100644 --- a/src/detect-tls-cert-validity.c +++ b/src/detect-tls-cert-validity.c @@ -123,9 +123,6 @@ void DetectTlsValidityRegister (void) DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); - DetectAppLayerInspectEngineRegister("tls_validity", ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectGenericList, NULL); - g_tls_validity_buffer_id = DetectBufferTypeGetByName("tls_validity"); } From 9db5e02a8f2477ec9364d1be9b38c541fae79ce2 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 28 Dec 2023 22:11:56 +0100 Subject: [PATCH 06/17] detect: move non-pf rules into special prefilter engines Instead of having a per detection engine list of rule that couldn't be prefiltered, put those into special "prefilter" engines. For packet and frame rules this doesn't change much, it just removes some hard coded logic from the detect engine. For the packet non-prefilter rules in the "non-prefilter" special prefilter engine, add additional filtering for the packet variant. It can prefilter on alproto, dsize and dest port. The frame non-prefilter rules are added to a single engine, that per rule checks the alproto and the type. For app-layer, there is an engine per progress value, per app-layer protocol and per direction. This hooks app-layer non-prefilter rules into the app inspect logic at the correct "progress" hook. e.g. a rule like dns.query; bsize:1; Negated MPM rules will also fall into this category: dns.query; content:!"abc"; Are part of a special "generic list" app engine for dns, at the same progress hook as `dns.query`. This all results in a lot fewer checks: previous: -------------------------------------------------------------------------- Date: 1/29/2025 -- 10:22:25. Sorted by: number of checks. -------------------------------------------------------------------------- Num Rule Gid Rev Ticks % Checks Matches Max Ticks Avg Ticks Avg Match Avg No Match -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- -------------- 1 20 1 0 181919672 11.85 588808 221 60454 308.96 2691.46 308.07 2 50 1 0 223455914 14.56 453104 418 61634 493.17 3902.59 490.02 3 60 1 0 185990683 12.12 453104 418 60950 410.48 1795.40 409.20 4 51 1 0 192436011 12.54 427028 6084 61223 450.64 2749.12 417.42 5 61 1 0 180401533 11.75 427028 6084 61093 422.46 2177.04 397.10 6 70 1 0 153899099 10.03 369836 0 61282 416.13 0.00 416.13 7 71 1 0 123389405 8.04 369836 12833 44921 333.63 2430.23 258.27 8 41 1 0 63889876 4.16 155824 12568 39138 410.01 1981.97 272.10 9 40 1 0 64149724 4.18 155818 210 39792 411.70 4349.57 406.38 10 10 1 0 70848850 4.62 65558 0 39544 1080.70 0.00 1080.70 11 11 1 0 94743878 6.17 65558 32214 60547 1445.19 2616.14 313.92 this commit: -------------------------------------------------------------------------- Date: 1/29/2025 -- 10:15:46. Sorted by: number of checks. -------------------------------------------------------------------------- Num Rule Gid Rev Ticks % Checks Matches Max Ticks Avg Ticks Avg Match Avg No Match -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- -------------- 1 50 1 0 138776766 19.23 95920 418 167584 1446.80 3953.11 1435.83 2 60 1 0 97988084 13.58 95920 418 182817 1021.56 1953.63 1017.48 3 51 1 0 105318318 14.60 69838 6084 65649 1508.04 2873.38 1377.74 4 61 1 0 89571260 12.41 69838 6084 164632 1282.56 2208.41 1194.20 5 11 1 0 91132809 12.63 32779 32214 373569 2780.22 2785.58 2474.45 6 10 1 0 66095303 9.16 32779 0 56704 2016.39 0.00 2016.39 7 70 1 0 48107573 6.67 12928 0 42832 3721.19 0.00 3721.19 8 71 1 0 32308792 4.48 12928 12833 39565 2499.13 2510.05 1025.09 9 41 1 0 25546837 3.54 12886 12470 41479 1982.53 1980.84 2033.05 10 40 1 0 26069992 3.61 12886 210 38495 2023.13 4330.05 1984.91 11 20 1 0 639025 0.09 221 221 14750 2891.52 2891.52 0.00 --- src/app-layer-frames.h | 2 + src/decode.h | 4 + src/detect-engine-build.c | 4 +- src/detect-engine-frame.c | 5 +- src/detect-engine-prefilter.c | 524 +++++++++++++++++++++++++++++++- src/detect-engine-prefilter.h | 3 +- src/detect-engine-profile.c | 6 +- src/detect-engine-siggroup.c | 88 ------ src/detect-engine.c | 16 +- src/detect.c | 196 ++---------- src/detect.h | 24 +- src/packet.c | 1 + src/util-profiling-rulegroups.c | 3 +- 13 files changed, 562 insertions(+), 314 deletions(-) diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h index f1d5135d8953..e2eb2d461236 100644 --- a/src/app-layer-frames.h +++ b/src/app-layer-frames.h @@ -26,6 +26,8 @@ #include "rust.h" +/** special value for matching any type */ +#define FRAME_ANY_TYPE 62 /** max 63 to fit the 64 bit per protocol space */ #define FRAME_STREAM_TYPE 63 diff --git a/src/decode.h b/src/decode.h index c74a94cc6374..387be58756b6 100644 --- a/src/decode.h +++ b/src/decode.h @@ -94,6 +94,9 @@ enum PktSrcEnum { #include "util-validate.h" +/* for now a uint8_t is enough -- here in decode as it's part of the packet */ +#define SignatureMask uint8_t + /* forward declarations */ struct DetectionEngineThreadCtx_; typedef struct AppLayerThreadCtx_ AppLayerThreadCtx; @@ -508,6 +511,7 @@ typedef struct Packet_ /* coccinelle: Packet:flowflags:FLOW_PKT_ */ uint8_t app_update_direction; // enum StreamUpdateDir + SignatureMask sig_mask; /* Pkt Flags */ uint32_t flags; diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index fd73520ba9c1..1793486ab1db 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -1983,8 +1983,6 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx) PrefilterSetupRuleGroup(de_ctx, sgh); - SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh); - sgh->id = idx; cnt++; } @@ -1995,7 +1993,7 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx) if (de_ctx->decoder_event_sgh != NULL) { /* no need to set filestore count here as that would make a * signature not decode event only. */ - SigGroupHeadBuildNonPrefilterArray(de_ctx, de_ctx->decoder_event_sgh); + PrefilterSetupRuleGroup(de_ctx, de_ctx->decoder_event_sgh); } int dump_grouping = 0; diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c index fd3163d59732..85c64772aac1 100644 --- a/src/detect-engine-frame.c +++ b/src/detect-engine-frame.c @@ -76,8 +76,9 @@ void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt); PrefilterEngine *engine = sgh->frame_engines; do { - BUG_ON(engine->alproto == ALPROTO_UNKNOWN); - if (engine->alproto == alproto && engine->ctx.frame_type == frame->type) { + if ((engine->alproto == alproto || engine->alproto == ALPROTO_UNKNOWN) && + (engine->ctx.frame_type == frame->type || + engine->ctx.frame_type == FRAME_ANY_TYPE)) { SCLogDebug("frame %p engine %p", frame, engine); PREFILTER_PROFILING_START(det_ctx); engine->cb.PrefilterFrame(det_ctx, engine->pectx, p, frames, frame); diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index db388bd6d70d..cc0fa4a17c12 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -51,12 +51,14 @@ #include "detect-engine-prefilter.h" #include "detect-engine-mpm.h" #include "detect-engine-frame.h" +#include "detect-engine-uint.h" #include "app-layer-parser.h" #include "app-layer-htp.h" #include "util-profiling.h" #include "util-validate.h" +#include "util-hash-string.h" static int PrefilterStoreGetId(DetectEngineCtx *de_ctx, const char *name, void (*FreeFunc)(void *)); @@ -127,6 +129,9 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, PREFILTER_PROFILING_END(det_ctx, engine->gid); if (tx->tx_progress > engine->ctx.tx_min_progress && engine->is_last_for_progress) { + SCLogDebug("tx->tx_progress %d engine->ctx.tx_min_progress %d " + "engine->is_last_for_progress %d", + tx->tx_progress, engine->ctx.tx_min_progress, engine->is_last_for_progress); tx->prefilter_flags |= BIT_U64(engine->ctx.tx_min_progress); } next: @@ -413,7 +418,198 @@ static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b) } } -void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +/** prefilter engine data for the non-prefilter engine for the prefilter API */ +struct PrefilterNonPFDataSig { + uint32_t sid : 30; + uint32_t type : 2; /**< type for `value` field below: 0:alproto 1:dport 2:dsize */ + uint16_t value; + /* since we have 2 more bytes available due to padding, we can add some additional + * filters here. */ + union { + struct { + SignatureMask sig_mask; + } pkt; + struct { + /* filter for frame type */ + uint8_t type; + } frame; + struct { + uint8_t foo; // TODO unused + } app; + }; +}; + +struct PrefilterNonPFData { + uint32_t size; + struct PrefilterNonPFDataSig array[]; +}; + +/** \internal + * \brief wrapper for use in APIs */ +static void PrefilterNonPFDataFree(void *data) +{ + SCFree(data); +} + +static void PrefilterTxNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f, + void *tx, const uint64_t tx_id, const AppLayerTxData *tx_data, const uint8_t flags) +{ + const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx; + SCLogDebug("adding %u sids", data->size); + for (uint32_t i = 0; i < data->size; i++) { + const struct PrefilterNonPFDataSig *ds = &data->array[i]; + DEBUG_VALIDATE_BUG_ON(ds->value == ALPROTO_UNKNOWN); + DEBUG_VALIDATE_BUG_ON(!AppProtoEquals(ds->value, f->alproto)); + const uint32_t sid = ds->sid; + PrefilterAddSids(&det_ctx->pmq, &sid, 1); + } +} + +#ifdef NONPF_PKT_STATS +static thread_local uint64_t prefilter_pkt_nonpf_called = 0; +static thread_local uint64_t prefilter_pkt_nonpf_mask_fail = 0; +static thread_local uint64_t prefilter_pkt_nonpf_alproto_fail = 0; +static thread_local uint64_t prefilter_pkt_nonpf_dsize_fail = 0; +static thread_local uint64_t prefilter_pkt_nonpf_dport_fail = 0; +static thread_local uint64_t prefilter_pkt_nonpf_sids = 0; +#define NONPF_PKT_STATS_INCR(s) (s)++ +#else +#define NONPF_PKT_STATS_INCR(s) +#endif + +void PrefilterPktNonPFStatsDump(void) +{ +#ifdef NONPF_PKT_STATS + SCLogDebug("prefilter non-pf: called:%" PRIu64 ", mask_fail:%" PRIu64 ", alproto fail:%" PRIu64 + ", dport fail:%" PRIu64 ", dsize fail:%" PRIu64 ", sids:%" PRIu64 + ", avg sids:%" PRIu64, + prefilter_pkt_nonpf_called, prefilter_pkt_nonpf_mask_fail, + prefilter_pkt_nonpf_alproto_fail, prefilter_pkt_nonpf_dport_fail, + prefilter_pkt_nonpf_dsize_fail, prefilter_pkt_nonpf_sids, + prefilter_pkt_nonpf_called ? prefilter_pkt_nonpf_sids / prefilter_pkt_nonpf_called : 0); +#endif +} + +static void PrefilterPktNonPF(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + const uint16_t alproto = p->flow ? p->flow->alproto : ALPROTO_UNKNOWN; + const SignatureMask mask = p->sig_mask; + const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx; + SCLogDebug("adding %u sids", data->size); + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_called); + for (uint32_t i = 0; i < data->size; i++) { + const struct PrefilterNonPFDataSig *ds = &data->array[i]; + const SignatureMask rule_mask = ds->pkt.sig_mask; + if ((rule_mask & mask) == rule_mask) { + switch (ds->type) { + case 0: + if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) { + const uint32_t sid = ds->sid; + PrefilterAddSids(&det_ctx->pmq, &sid, 1); + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids); + } else { + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_alproto_fail); + } + break; + case 1: + if (ds->value == p->dp) { + const uint32_t sid = ds->sid; + PrefilterAddSids(&det_ctx->pmq, &sid, 1); + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids); + } else { + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_dport_fail); + } + break; + case 2: + if (ds->value == p->payload_len) { + const uint32_t sid = ds->sid; + PrefilterAddSids(&det_ctx->pmq, &sid, 1); + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids); + } else { + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_dsize_fail); + } + break; + } + } else { + NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_mask_fail); + } + } +} + +/** \internal + * \brief engine to select the non-prefilter rules for frames + * Checks the alproto and type as well. + * Direction needs no checking as the rule groups are per direction. */ +static void PrefilterFrameNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, + const Frames *frames, const Frame *frame) +{ + DEBUG_VALIDATE_BUG_ON(p->flow == NULL); + const uint16_t alproto = p->flow->alproto; + const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx; + SCLogDebug("adding %u sids", data->size); + for (uint32_t i = 0; i < data->size; i++) { + const struct PrefilterNonPFDataSig *ds = &data->array[i]; + if (ds->frame.type == frame->type && + (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto))) { + const uint32_t sid = ds->sid; + PrefilterAddSids(&det_ctx->pmq, &sid, 1); + } + } +} + +/* helper funcs for the non prefilter names hash */ + +static uint32_t NonPFNamesHash(HashTable *h, void *data, uint16_t _len) +{ + const char *str = data; + return StringHashDjb2((const uint8_t *)str, (uint16_t)strlen(str)) % h->array_size; +} + +static char NonPFNamesCompare(void *data1, uint16_t _len1, void *data2, uint16_t len2) +{ + const char *s1 = data1; + const char *s2 = data2; + return StringHashCompareFunc(data1, (uint16_t)strlen(s1), data2, (uint16_t)strlen(s2)); +} + +static void NonPFNamesFree(void *data) +{ + SCFree(data); +} + +/* helper funcs for assembling non-prefilter engines */ + +struct TxNonPFData { + AppProto alproto; + int dir; /**< 0: toserver, 1: toclient */ + // int sm_list; /**< sigmatch list for this engine */ + int progress; /**< progress state value to register at */ + uint32_t sigs_cnt; + struct PrefilterNonPFDataSig *sigs; + const char *engine_name; /**< pointer to name owned by DetectEngineCtx::non_pf_engine_names */ +}; + +static uint32_t TxNonPFHash(HashListTable *h, void *data, uint16_t _len) +{ + struct TxNonPFData *d = data; + return (d->alproto + d->progress + d->dir) % h->array_size; +} + +static char TxNonPFCompare(void *data1, uint16_t _len1, void *data2, uint16_t len2) +{ + struct TxNonPFData *d1 = data1; + struct TxNonPFData *d2 = data2; + return d1->alproto == d2->alproto && d1->progress == d2->progress && d1->dir == d2->dir; +} + +static void TxNonPFFree(void *data) +{ + struct TxNonPFData *d = data; + SCFree(d->sigs); + SCFree(d); +} + +int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { int r = PatternMatchPrepareGroup(de_ctx, sgh); if (r != 0) { @@ -434,6 +630,315 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } } + const uint32_t max_sids = DetectEngineGetMaxSigId(de_ctx); + SCLogDebug("max_sids %u", max_sids); + struct PrefilterNonPFDataSig *pkt_non_pf_array = SCCalloc(max_sids, sizeof(*pkt_non_pf_array)); + if (pkt_non_pf_array == NULL) { + return -1; + } + uint32_t pkt_non_pf_array_size = 0; + struct PrefilterNonPFDataSig *frame_non_pf_array = + SCCalloc(max_sids, sizeof(*frame_non_pf_array)); + if (frame_non_pf_array == NULL) { + SCFree(pkt_non_pf_array); + return -1; + } + uint32_t frame_non_pf_array_size = 0; + + HashListTable *tx_engines_hash = + HashListTableInit(256, TxNonPFHash, TxNonPFCompare, TxNonPFFree); + if (tx_engines_hash == NULL) { + SCFree(pkt_non_pf_array); + SCFree(frame_non_pf_array); + return -1; + } + + if (de_ctx->non_pf_engine_names == NULL) { + de_ctx->non_pf_engine_names = + HashTableInit(512, NonPFNamesHash, NonPFNamesCompare, NonPFNamesFree); + if (de_ctx->non_pf_engine_names == NULL) { + SCFree(pkt_non_pf_array); + SCFree(frame_non_pf_array); + HashListTableFree(tx_engines_hash); + return -1; + } + } + + SignatureMask pkt_mask = 0; + bool pkt_mask_init = false; +#ifdef NONPF_PKT_STATS + uint32_t nonpf_pkt_alproto = 0; + uint32_t nonpf_pkt_dsize = 0; + uint32_t nonpf_pkt_dport = 0; +#endif + const int app_events_list_id = DetectBufferTypeGetByName("app-layer-events"); + SCLogDebug("app_events_list_id %d", app_events_list_id); + for (uint32_t sig = 0; sig < sgh->init->sig_cnt; sig++) { + Signature *s = sgh->init->match_array[sig]; + if (s == NULL) + continue; + SCLogDebug("checking sid %u for non-prefilter", s->id); + if (s->init_data->mpm_sm != NULL && (s->flags & SIG_FLAG_MPM_NEG) == 0) + continue; + if (s->init_data->prefilter_sm != NULL) + continue; + if ((s->flags & (SIG_FLAG_PREFILTER | SIG_FLAG_MPM_NEG)) == SIG_FLAG_PREFILTER) + continue; + SCLogDebug("setting up sid %u for non-prefilter", s->id); + + uint8_t frame_type = 0; /**< only a single type per rule */ + bool tx_non_pf = false; + bool frame_non_pf = false; + bool pkt_non_pf = false; + for (uint32_t x = 0; x < s->init_data->buffer_index; x++) { + const int list_id = s->init_data->buffers[x].id; + const DetectBufferType *buf = DetectEngineBufferTypeGetById(de_ctx, list_id); + /* for now, exclude app-layer-events, as they are not tied to a specific + * progress value like other keywords. */ + SCLogDebug("list_id %d buf %p", list_id, buf); + if (list_id == app_events_list_id) + continue; + if (buf->packet) { + SCLogDebug("packet buf"); + /* packet is handled below */ + pkt_non_pf = true; + } else if (buf->frame) { + for (DetectEngineFrameInspectionEngine *f = de_ctx->frame_inspect_engines; + f != NULL; f = f->next) { + if (!((((s->flags & SIG_FLAG_TOSERVER) != 0 && f->dir == 0) || + ((s->flags & SIG_FLAG_TOCLIENT) != 0 && f->dir == 1)) && + list_id == (int)f->sm_list && + AppProtoEquals(s->alproto, f->alproto))) + continue; + + SCLogDebug("frame '%s' type %u", buf->name, f->type); + frame_type = f->type; + frame_non_pf = true; + + frame_non_pf_array[frame_non_pf_array_size].sid = s->num; + frame_non_pf_array[frame_non_pf_array_size].value = s->alproto; + frame_non_pf_array[frame_non_pf_array_size].frame.type = frame_type; + frame_non_pf_array_size++; + break; + } + + } else { + SCLogDebug("x %u list_id %d", x, list_id); + for (DetectEngineAppInspectionEngine *app = de_ctx->app_inspect_engines; + app != NULL; app = app->next) { + SCLogDebug("app %p proto %s list_d %d sig dir %0x", app, + AppProtoToString(app->alproto), app->sm_list, + s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)); + + /* skip if: + * - not in our dir + * - not our list + * - app proto mismatch. Both sig and app can have proto or unknown */ + if (!((((s->flags & SIG_FLAG_TOSERVER) != 0 && app->dir == 0) || + ((s->flags & SIG_FLAG_TOCLIENT) != 0 && app->dir == 1)) && + list_id == (int)app->sm_list && + (s->alproto == ALPROTO_UNKNOWN || app->alproto == ALPROTO_UNKNOWN || + AppProtoEquals(s->alproto, app->alproto)))) + continue; + + struct TxNonPFData lookup = { + .alproto = app->alproto, + .dir = app->dir, + //.sm_list = app->sm_list, + .progress = app->progress, + .sigs_cnt = 0, + .sigs = NULL, + .engine_name = NULL, + }; + struct TxNonPFData *e = HashListTableLookup(tx_engines_hash, &lookup, 0); + if (e != NULL) { + bool found = false; + // avoid adding same sid multiple times + for (uint32_t y = 0; y < e->sigs_cnt; y++) { + // BUG_ON(e->sigs[x].sid == s->num); + if (e->sigs[y].sid == s->num) { + found = true; + break; + } + } + if (!found) { + BUG_ON(e->sigs_cnt == max_sids); + e->sigs[e->sigs_cnt].sid = s->num; + e->sigs[e->sigs_cnt].value = app->alproto; + e->sigs_cnt++; + } + } else { + struct TxNonPFData *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) { + goto error; + } + add->dir = app->dir; + add->alproto = app->alproto; + // add->sm_list = app->sm_list; + add->progress = app->progress; + add->sigs = SCCalloc(max_sids, sizeof(struct PrefilterNonPFDataSig)); + if (add->sigs == NULL) { + SCFree(add); + goto error; + } + add->sigs_cnt = 0; + add->sigs[add->sigs_cnt].sid = s->num; + add->sigs[add->sigs_cnt].value = app->alproto; + add->sigs_cnt++; + + char engine_name[80]; + snprintf(engine_name, sizeof(engine_name), "%s:%s:non_pf:%s", + AppProtoToString(app->alproto), buf->name, + app->dir == 0 ? "toserver" : "toclient"); + char *engine_name_heap = SCStrdup(engine_name); + if (engine_name_heap == NULL) { + SCFree(add->sigs); + SCFree(add); + goto error; + } + int result = HashTableAdd(de_ctx->non_pf_engine_names, engine_name_heap, + (uint16_t)strlen(engine_name_heap)); + if (result != 0) { + SCFree(add->sigs); + SCFree(add); + goto error; + } + + add->engine_name = engine_name_heap; + SCLogDebug("engine_name_heap %s", engine_name_heap); + + int ret = HashListTableAdd(tx_engines_hash, add, 0); + if (ret != 0) { + SCFree(add->sigs); + SCFree(add); + goto error; + } + } + tx_non_pf = true; + // break; + } + } + } + /* mark as prefiltered as the sig is now part of a engine */ + // s->flags |= SIG_FLAG_PREFILTER; + // TODO doesn't work for sigs that are in multiple sgh's + + /* default to pkt if there was no tx or frame match */ + if (!(tx_non_pf || frame_non_pf)) { + if (!pkt_non_pf) { + SCLogDebug("not frame, not tx, so pkt"); + } + pkt_non_pf = true; + } + + SCLogDebug("setting up sid %u for non-prefilter: %s", s->id, + tx_non_pf ? "tx engine" : (frame_non_pf ? "frame engine" : "pkt engine")); + + if (pkt_non_pf) { + /* for pkt non prefilter, we have some space in the structure, + * so we can squeeze another filter */ + uint8_t type; + uint16_t value; + if ((s->flags & SIG_FLAG_DSIZE) && s->dsize_mode == DETECT_UINT_EQ) { + SCLogDebug("dsize extra match"); + type = 2; + value = s->dsize_low; +#ifdef NONPF_PKT_STATS + nonpf_pkt_dsize++; +#endif + } else if (s->dp != NULL && s->dp->next == NULL && s->dp->port == s->dp->port2) { + type = 1; + value = s->dp->port; +#ifdef NONPF_PKT_STATS + nonpf_pkt_dport++; +#endif + } else { + type = 0; + value = s->alproto; +#ifdef NONPF_PKT_STATS + nonpf_pkt_alproto++; +#endif + } + + pkt_non_pf_array[pkt_non_pf_array_size].sid = s->num; + pkt_non_pf_array[pkt_non_pf_array_size].value = value; + pkt_non_pf_array[pkt_non_pf_array_size].type = type; + pkt_non_pf_array[pkt_non_pf_array_size].pkt.sig_mask = s->mask; + pkt_non_pf_array_size++; + + if (pkt_mask_init) { + pkt_mask &= s->mask; + } else { + pkt_mask = s->mask; + pkt_mask_init = true; + } + } + } + + /* for each unique sig set, add an engine */ + for (HashListTableBucket *b = HashListTableGetListHead(tx_engines_hash); b != NULL; + b = HashListTableGetListNext(b)) { + struct TxNonPFData *t = HashListTableGetListData(b); + SCLogDebug("%s engine for %s hook %d has %u non-pf sigs", + t->dir == 0 ? "toserver" : "toclient", AppProtoToString(t->alproto), t->progress, + t->sigs_cnt); + + if (((sgh->init->direction & SIG_FLAG_TOSERVER) && t->dir == 1) || + ((sgh->init->direction & SIG_FLAG_TOCLIENT) && t->dir == 0)) { + SCLogDebug("skipped"); + continue; + } + + struct PrefilterNonPFData *data = + SCCalloc(1, sizeof(*data) + t->sigs_cnt * sizeof(data->array[0])); + if (data == NULL) + goto error; + data->size = t->sigs_cnt; + memcpy((uint8_t *)&data->array, t->sigs, t->sigs_cnt * sizeof(data->array[0])); + if (PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxNonPF, t->alproto, t->progress, + (void *)data, PrefilterNonPFDataFree, t->engine_name) < 0) { + SCFree(data); + goto error; + } + } + HashListTableFree(tx_engines_hash); + tx_engines_hash = NULL; + + if (pkt_non_pf_array_size) { + struct PrefilterNonPFData *data = + SCCalloc(1, sizeof(*data) + pkt_non_pf_array_size * sizeof(data->array[0])); + if (data == NULL) + goto error; + data->size = pkt_non_pf_array_size; + memcpy((uint8_t *)&data->array, pkt_non_pf_array, + pkt_non_pf_array_size * sizeof(data->array[0])); + if (PrefilterAppendEngine(de_ctx, sgh, PrefilterPktNonPF, pkt_mask, (void *)data, + PrefilterNonPFDataFree, "packet:non_pf") < 0) { + SCFree(data); + goto error; + } + } + if (frame_non_pf_array_size) { + SCLogDebug("%u frame non-pf sigs", frame_non_pf_array_size); + struct PrefilterNonPFData *data = + SCCalloc(1, sizeof(*data) + frame_non_pf_array_size * sizeof(data->array[0])); + if (data == NULL) + goto error; + data->size = frame_non_pf_array_size; + memcpy((uint8_t *)&data->array, frame_non_pf_array, + frame_non_pf_array_size * sizeof(data->array[0])); + if (PrefilterAppendFrameEngine(de_ctx, sgh, PrefilterFrameNonPF, ALPROTO_UNKNOWN, + FRAME_ANY_TYPE, (void *)data, PrefilterNonPFDataFree, "frame:non_pf") < 0) { + SCFree(data); + goto error; + } + } + + SCFree(pkt_non_pf_array); + pkt_non_pf_array = NULL; + SCFree(frame_non_pf_array); + frame_non_pf_array = NULL; + /* we have lists of engines in sgh->init now. Lets setup the * match arrays */ PrefilterEngineList *el; @@ -444,7 +949,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } sgh->pkt_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); if (sgh->pkt_engines == NULL) { - return; + goto error; } memset(sgh->pkt_engines, 0x00, (cnt * sizeof(PrefilterEngine))); @@ -469,7 +974,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } sgh->payload_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); if (sgh->payload_engines == NULL) { - return; + goto error; } memset(sgh->payload_engines, 0x00, (cnt * sizeof(PrefilterEngine))); @@ -494,7 +999,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } sgh->tx_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); if (sgh->tx_engines == NULL) { - return; + goto error; } memset(sgh->tx_engines, 0x00, (cnt * sizeof(PrefilterEngine))); @@ -570,7 +1075,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } sgh->frame_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); if (sgh->frame_engines == NULL) { - return; + goto error; } memset(sgh->frame_engines, 0x00, (cnt * sizeof(PrefilterEngine))); @@ -589,6 +1094,15 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) e++; } } + return 0; + +error: + if (tx_engines_hash) { + HashListTableFree(tx_engines_hash); + } + SCFree(pkt_non_pf_array); + SCFree(frame_non_pf_array); + return -1; } /* hash table for assigning a unique id to each engine type. */ diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h index ec58594b9b00..04882ec633d7 100644 --- a/src/detect-engine-prefilter.h +++ b/src/detect-engine-prefilter.h @@ -72,7 +72,7 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, void PrefilterFreeEnginesList(PrefilterEngineList *list); -void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh); +int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh); #ifdef PROFILING @@ -91,4 +91,5 @@ int PrefilterMultiGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectBufferMpmRegistry *mpm_reg, int list_id); +void PrefilterPktNonPFStatsDump(void); #endif diff --git a/src/detect-engine-profile.c b/src/detect-engine-profile.c index 36cf6e106b05..9f46fdac6ec2 100644 --- a/src/detect-engine-profile.c +++ b/src/detect-engine-profile.c @@ -47,7 +47,8 @@ void RulesDumpTxMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupH jb_set_uint(js, "rule_group_id", sgh->id); jb_set_uint(js, "rule_cnt", rule_cnt); jb_set_uint(js, "pkt_rule_cnt", pkt_prefilter_cnt); - jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt); + // TODO + // jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt); jb_open_array(js, "rules"); for (uint32_t x = 0; x < rule_cnt; x++) { @@ -91,7 +92,8 @@ void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, jb_set_string(js, "inspect_type", "packet"); jb_set_uint(js, "rule_group_id", sgh->id); jb_set_uint(js, "rule_cnt", det_ctx->match_array_cnt); - jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt); + // TODO + // jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt); jb_open_array(js, "rules"); for (uint32_t x = 0; x < det_ctx->match_array_cnt; x++) { diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 49982f028b28..9c8364d403c3 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -166,18 +166,6 @@ void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) SCLogDebug("sgh %p", sgh); - if (sgh->non_pf_other_store_array != NULL) { - SCFree(sgh->non_pf_other_store_array); - sgh->non_pf_other_store_array = NULL; - sgh->non_pf_other_store_cnt = 0; - } - - if (sgh->non_pf_syn_store_array != NULL) { - SCFree(sgh->non_pf_syn_store_array); - sgh->non_pf_syn_store_array = NULL; - sgh->non_pf_syn_store_cnt = 0; - } - if (sgh->init != NULL) { SigGroupHeadInitDataFree(sgh->init); sgh->init = NULL; @@ -619,82 +607,6 @@ void SigGroupHeadSetupFiles(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) } } -/** \brief build an array of rule id's for sigs with no prefilter - * Also updated de_ctx::non_pf_store_cnt_max to track the highest cnt - */ -int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) -{ - Signature *s = NULL; - uint32_t sig = 0; - uint32_t non_pf = 0; - uint32_t non_pf_syn = 0; - - if (sgh == NULL) - return 0; - - BUG_ON(sgh->non_pf_other_store_array != NULL); - - for (sig = 0; sig < sgh->init->sig_cnt; sig++) { - s = sgh->init->match_array[sig]; - if (s == NULL) - continue; - - if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) { - if (!(DetectFlagsSignatureNeedsSynPackets(s))) { - non_pf++; - } - non_pf_syn++; - } - } - - if (non_pf == 0 && non_pf_syn == 0) { - sgh->non_pf_other_store_array = NULL; - sgh->non_pf_syn_store_array = NULL; - return 0; - } - - if (non_pf > 0) { - sgh->non_pf_other_store_array = SCCalloc(non_pf, sizeof(SignatureNonPrefilterStore)); - BUG_ON(sgh->non_pf_other_store_array == NULL); - } - - if (non_pf_syn > 0) { - sgh->non_pf_syn_store_array = SCCalloc(non_pf_syn, sizeof(SignatureNonPrefilterStore)); - BUG_ON(sgh->non_pf_syn_store_array == NULL); - } - - for (sig = 0; sig < sgh->init->sig_cnt; sig++) { - s = sgh->init->match_array[sig]; - if (s == NULL) - continue; - - if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) { - if (!(DetectFlagsSignatureNeedsSynPackets(s))) { - BUG_ON(sgh->non_pf_other_store_cnt >= non_pf); - BUG_ON(sgh->non_pf_other_store_array == NULL); - sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].id = s->num; - sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].mask = s->mask; - sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].alproto = s->alproto; - sgh->non_pf_other_store_cnt++; - } - - BUG_ON(sgh->non_pf_syn_store_cnt >= non_pf_syn); - BUG_ON(sgh->non_pf_syn_store_array == NULL); - sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].id = s->num; - sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].mask = s->mask; - sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].alproto = s->alproto; - sgh->non_pf_syn_store_cnt++; - } - } - - /* track highest cnt for any sgh in our de_ctx */ - uint32_t max = MAX(sgh->non_pf_other_store_cnt, sgh->non_pf_syn_store_cnt); - if (max > de_ctx->non_pf_store_cnt_max) - de_ctx->non_pf_store_cnt_max = max; - - return 0; -} - /** * \brief Check if a SigGroupHead contains a Signature, whose sid is sent as an * argument. diff --git a/src/detect-engine.c b/src/detect-engine.c index d3d8077f8518..58de4691ed32 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2720,6 +2720,9 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) SCDetectRequiresStatusFree(de_ctx->requirements); } + if (de_ctx->non_pf_engine_names) { + HashTableFree(de_ctx->non_pf_engine_names); + } SCFree(de_ctx); //DetectAddressGroupPrintMemory(); //DetectSigGroupPrintMemory(); @@ -3286,13 +3289,6 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx * return TM_ECODE_FAILED; } - /* sized to the max of our sgh settings. A max setting of 0 implies that all - * sgh's have: sgh->non_pf_store_cnt == 0 */ - if (de_ctx->non_pf_store_cnt_max > 0) { - det_ctx->non_pf_id_array = SCCalloc(de_ctx->non_pf_store_cnt_max, sizeof(SigIntId)); - BUG_ON(det_ctx->non_pf_id_array == NULL); - } - /* DeState */ if (de_ctx->sig_array_len > 0) { det_ctx->match_array_len = de_ctx->sig_array_len; @@ -3545,10 +3541,6 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) if (det_ctx->spm_thread_ctx != NULL) { SpmDestroyThreadCtx(det_ctx->spm_thread_ctx); } - - if (det_ctx->non_pf_id_array != NULL) - SCFree(det_ctx->non_pf_id_array); - if (det_ctx->match_array != NULL) SCFree(det_ctx->match_array); @@ -3599,7 +3591,7 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) } AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events); - + PrefilterPktNonPFStatsDump(); SCFree(det_ctx); ThresholdCacheThreadFree(); diff --git a/src/detect.c b/src/detect.c index 19a569de1a39..e4be96958eb7 100644 --- a/src/detect.c +++ b/src/detect.c @@ -71,7 +71,6 @@ typedef struct DetectRunScratchpad { const uint8_t flow_flags; /* flow/state flags: STREAM_* */ const bool app_decoder_events; const SigGroupHead *sgh; - SignatureMask pkt_mask; } DetectRunScratchpad; /* prototypes */ @@ -264,118 +263,18 @@ const SigGroupHead *SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, SCReturnPtr(sgh, "SigGroupHead"); } -static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx, - DetectEngineThreadCtx *det_ctx) +static inline void DetectPrefilterCopyDeDup(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx) { - SigIntId mpm, nonmpm; - SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array; - SigIntId *nonmpm_ptr = det_ctx->non_pf_id_array; - uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt; - uint32_t n_cnt = det_ctx->non_pf_id_cnt; - SigIntId *final_ptr; - uint32_t final_cnt; - SigIntId id; - SigIntId previous_id = (SigIntId)-1; + SigIntId *pf_ptr = det_ctx->pmq.rule_id_array; + uint32_t final_cnt = det_ctx->pmq.rule_id_array_cnt; Signature **sig_array = de_ctx->sig_array; Signature **match_array = det_ctx->match_array; - Signature *s; - - SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt); - - /* Load first values. */ - if (likely(m_cnt)) { - mpm = *mpm_ptr; - } else { - /* mpm list is empty */ - final_ptr = nonmpm_ptr; - final_cnt = n_cnt; - goto final; - } - if (likely(n_cnt)) { - nonmpm = *nonmpm_ptr; - } else { - /* non-mpm list is empty. */ - final_ptr = mpm_ptr; - final_cnt = m_cnt; - goto final; - } - while (1) { - if (mpm < nonmpm) { - /* Take from mpm list */ - id = mpm; - - s = sig_array[id]; - /* As the mpm list can contain duplicates, check for that here. */ - if (likely(id != previous_id)) { - *match_array++ = s; - previous_id = id; - } - if (unlikely(--m_cnt == 0)) { - /* mpm list is now empty */ - final_ptr = nonmpm_ptr; - final_cnt = n_cnt; - goto final; - } - mpm_ptr++; - mpm = *mpm_ptr; - } else if (mpm > nonmpm) { - id = nonmpm; - - s = sig_array[id]; - /* As the mpm list can contain duplicates, check for that here. */ - if (likely(id != previous_id)) { - *match_array++ = s; - previous_id = id; - } - if (unlikely(--n_cnt == 0)) { - final_ptr = mpm_ptr; - final_cnt = m_cnt; - goto final; - } - nonmpm_ptr++; - nonmpm = *nonmpm_ptr; - - } else { /* implied mpm == nonmpm */ - /* special case: if on both lists, it's a negated mpm pattern */ - - /* mpm list may have dups, so skip past them here */ - while (--m_cnt != 0) { - mpm_ptr++; - mpm = *mpm_ptr; - if (mpm != nonmpm) - break; - } - /* if mpm is done, update nonmpm_ptrs and jump to final */ - if (unlikely(m_cnt == 0)) { - n_cnt--; - - /* mpm list is now empty */ - final_ptr = ++nonmpm_ptr; - final_cnt = n_cnt; - goto final; - } - /* otherwise, if nonmpm is done jump to final for mpm - * mpm ptrs already updated */ - if (unlikely(--n_cnt == 0)) { - final_ptr = mpm_ptr; - final_cnt = m_cnt; - goto final; - } - - /* not at end of the lists, update nonmpm. Mpm already - * updated in while loop above. */ - nonmpm_ptr++; - nonmpm = *nonmpm_ptr; - } - } - - final: /* Only one list remaining. Just walk that list. */ - + SigIntId previous_id = (SigIntId)-1; while (final_cnt-- > 0) { - id = *final_ptr++; - s = sig_array[id]; + SigIntId id = *pf_ptr++; + Signature *s = sig_array[id]; - /* As the mpm list can contain duplicates, check for that here. */ + /* As the prefilter list can contain duplicates, check for that here. */ if (likely(id != previous_id)) { *match_array++ = s; previous_id = id; @@ -383,48 +282,10 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx, } det_ctx->match_array_cnt = match_array - det_ctx->match_array; - DEBUG_VALIDATE_BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_pf_id_cnt) < det_ctx->match_array_cnt); + DEBUG_VALIDATE_BUG_ON(det_ctx->pmq.rule_id_array_cnt < det_ctx->match_array_cnt); PMQ_RESET(&det_ctx->pmq); } -/** \internal - * \brief build non-prefilter list based on the rule group list we've set. - */ -static inline void DetectPrefilterBuildNonPrefilterList( - DetectEngineThreadCtx *det_ctx, const SignatureMask mask, const AppProto alproto) -{ - for (uint32_t x = 0; x < det_ctx->non_pf_store_cnt; x++) { - /* only if the mask matches this rule can possibly match, - * so build the non_mpm array only for match candidates */ - const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask; - const AppProto rule_alproto = det_ctx->non_pf_store_ptr[x].alproto; - if ((rule_mask & mask) == rule_mask && - (rule_alproto == 0 || AppProtoEquals(rule_alproto, alproto))) { - det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id; - } - } -} - -/** \internal - * \brief select non-mpm list - * Based on the packet properties, select the non-mpm list to use - * \todo move non_pf_store* into scratchpad */ -static inline void -DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch) -{ - if ((p->proto == IPPROTO_TCP) && PacketIsTCP(p) && (PacketGetTCP(p)->th_flags & TH_SYN)) { - det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array; - det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt; - } else { - det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array; - det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt; - } - SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)", - det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt, - scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt, - scratch->sgh->non_pf_other_store_array, scratch->sgh->non_pf_other_store_cnt); -} - /** \internal * \brief update flow's file tracking flags based on the detection engine * A set of flags is prepared that is sent to the File API. The @@ -676,42 +537,21 @@ static inline void DetectRunPrefilterPkt( DetectRunScratchpad *scratch ) { - DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch); - /* create our prefilter mask */ - PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events); - - /* build and prefilter non_pf list against the mask of the packet */ - PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST); - det_ctx->non_pf_id_cnt = 0; - if (likely(det_ctx->non_pf_store_cnt > 0)) { - DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto); - } - PACKET_PROFILING_DETECT_END(p, PROF_DETECT_NONMPMLIST); - + PacketCreateMask(p, &p->sig_mask, scratch->alproto, scratch->app_decoder_events); /* run the prefilter engines */ - Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, scratch->pkt_mask); + Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, p->sig_mask); /* create match list if we have non-pf and/or pf */ - if (det_ctx->non_pf_store_cnt || det_ctx->pmq.rule_id_array_cnt) { + if (det_ctx->pmq.rule_id_array_cnt) { #ifdef PROFILING if (tv) { StatsAddUI64(tv, det_ctx->counter_mpm_list, (uint64_t)det_ctx->pmq.rule_id_array_cnt); } #endif PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_SORT2); - DetectPrefilterMergeSort(de_ctx, det_ctx); + DetectPrefilterCopyDeDup(de_ctx, det_ctx); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_SORT2); } - -#ifdef PROFILING - if (tv) { - StatsAddUI64(tv, det_ctx->counter_nonmpm_list, - (uint64_t)det_ctx->non_pf_store_cnt); - /* non mpm sigs after mask prefilter */ - StatsAddUI64(tv, det_ctx->counter_fnonmpm_list, - (uint64_t)det_ctx->non_pf_id_cnt); - } -#endif } /** \internal @@ -808,8 +648,8 @@ static inline void DetectRulePacketRules( /* don't run mask check for stateful rules. * There we depend on prefilter */ - if ((s->mask & scratch->pkt_mask) != s->mask) { - SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask); + if ((s->mask & p->sig_mask) != s->mask) { + SCLogDebug("mask mismatch %x & %x != %x", s->mask, p->sig_mask, s->mask); goto next; } @@ -962,7 +802,7 @@ static DetectRunScratchpad DetectRunSetup( app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser); } - DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL, 0 }; + DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL }; PACKET_PROFILING_DETECT_END(p, PROF_DETECT_SETUP); return pad; } @@ -1094,14 +934,14 @@ DetectRunTxSortHelper(const void *a, const void *b) void *DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto, uint8_t flow_flags) { if (unlikely(alproto == ALPROTO_DOH2)) { - if (engine_alproto == ALPROTO_DNS) { + if (engine_alproto == ALPROTO_DNS) { // || engine_alproto == ALPROTO_UNKNOWN) { // need to get the dns tx pointer tx_ptr = SCDoH2GetDnsTx(tx_ptr, flow_flags); - } else if (engine_alproto != ALPROTO_HTTP2) { + } else if (engine_alproto != ALPROTO_HTTP2 && engine_alproto != ALPROTO_UNKNOWN) { // incompatible engine->alproto with flow alproto tx_ptr = NULL; } - } else if (engine_alproto != alproto) { + } else if (engine_alproto != alproto && engine_alproto != ALPROTO_UNKNOWN) { // incompatible engine->alproto with flow alproto tx_ptr = NULL; } diff --git a/src/detect.h b/src/detect.h index 2348c54e679d..ac7a0df3d764 100644 --- a/src/detect.h +++ b/src/detect.h @@ -309,9 +309,6 @@ typedef struct DetectPort_ { // vacancy 1x #define SIG_MASK_REQUIRE_ENGINE_EVENT BIT_U8(7) -/* for now a uint8_t is enough */ -#define SignatureMask uint8_t - #define FILE_SIG_NEED_FILE 0x01 #define FILE_SIG_NEED_FILENAME 0x02 #define FILE_SIG_NEED_MAGIC 0x04 /**< need the start of the file */ @@ -865,10 +862,6 @@ typedef struct DetectEngineCtx_ { uint32_t signum; - /** Maximum value of all our sgh's non_mpm_store_cnt setting, - * used to alloc det_ctx::non_mpm_id_array */ - uint32_t non_pf_store_cnt_max; - /* used by the signature ordering module */ struct SCSigOrderFunc_ *sc_sig_order_funcs; @@ -1053,6 +1046,8 @@ typedef struct DetectEngineCtx_ { /* number of signatures using filestore, limited as u16 */ uint16_t filestore_cnt; + + HashTable *non_pf_engine_names; } DetectEngineCtx; /* Engine groups profiles (low, medium, high, custom) */ @@ -1108,11 +1103,6 @@ typedef struct DetectEngineThreadCtx_ { /* the thread to which this detection engine thread belongs */ ThreadVars *tv; - /** Array of non-prefiltered sigs that need to be evaluated. Updated - * per packet based on the rule group and traffic properties. */ - SigIntId *non_pf_id_array; - uint32_t non_pf_id_cnt; // size is cnt * sizeof(uint32_t) - uint32_t mt_det_ctxs_cnt; struct DetectEngineThreadCtx_ **mt_det_ctxs; HashTable *mt_det_ctxs_hash; @@ -1200,9 +1190,6 @@ typedef struct DetectEngineThreadCtx_ { RuleMatchCandidateTx *tx_candidates; uint32_t tx_candidates_size; - SignatureNonPrefilterStore *non_pf_store_ptr; - uint32_t non_pf_store_cnt; - MpmThreadCtx mtc; /**< thread ctx for the mpm */ PrefilterRuleStore pmq; @@ -1474,13 +1461,6 @@ typedef struct SigGroupHead_ { uint32_t id; /**< unique id used to index sgh_array for stats */ - /* non prefilter list excluding SYN rules */ - uint32_t non_pf_other_store_cnt; - uint32_t non_pf_syn_store_cnt; - SignatureNonPrefilterStore *non_pf_other_store_array; // size is non_mpm_store_cnt * sizeof(SignatureNonPrefilterStore) - /* non mpm list including SYN rules */ - SignatureNonPrefilterStore *non_pf_syn_store_array; // size is non_mpm_syn_store_cnt * sizeof(SignatureNonPrefilterStore) - PrefilterEngine *pkt_engines; PrefilterEngine *payload_engines; PrefilterEngine *tx_engines; diff --git a/src/packet.c b/src/packet.c index cb6dcf618380..076642ac6f38 100644 --- a/src/packet.c +++ b/src/packet.c @@ -97,6 +97,7 @@ void PacketReinit(Packet *p) p->recursion_level = 0; PACKET_FREE_EXTDATA(p); p->app_update_direction = 0; + p->sig_mask = 0; p->flags = 0; p->flowflags = 0; p->pkt_src = 0; diff --git a/src/util-profiling-rulegroups.c b/src/util-profiling-rulegroups.c index b25e89b01bb4..7e4612fbf0ca 100644 --- a/src/util-profiling-rulegroups.c +++ b/src/util-profiling-rulegroups.c @@ -263,13 +263,14 @@ SCProfilingSghUpdateCounter(DetectEngineThreadCtx *det_ctx, const SigGroupHead * if (det_ctx != NULL && det_ctx->sgh_perf_data != NULL && sgh->id < det_ctx->de_ctx->sgh_array_cnt) { SCProfileSghData *p = &det_ctx->sgh_perf_data[sgh->id]; p->checks++; - +#if 0 // TODO if (det_ctx->non_pf_store_cnt > 0) { if (det_ctx->non_pf_store_ptr == sgh->non_pf_syn_store_array) p->non_mpm_syn++; else p->non_mpm_generic++; } +#endif p->post_prefilter_sigs_total += det_ctx->match_array_cnt; if (det_ctx->match_array_cnt > p->post_prefilter_sigs_max) p->post_prefilter_sigs_max = det_ctx->match_array_cnt; From 6ebe28905a0b879e1cb3525a9d009f3aee2a41c7 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 13 Jan 2025 20:46:58 +0100 Subject: [PATCH 07/17] app-layer: constify AppLayerGetProtoByName --- src/app-layer.c | 2 +- src/app-layer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app-layer.c b/src/app-layer.c index e5efcdf361d1..c5332c96a3a4 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -996,7 +996,7 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow * /***** Utility *****/ -AppProto AppLayerGetProtoByName(char *alproto_name) +AppProto AppLayerGetProtoByName(const char *alproto_name) { SCEnter(); AppProto r = AppLayerProtoDetectGetProtoByName(alproto_name); diff --git a/src/app-layer.h b/src/app-layer.h index 1dea87ee34a8..a80c8cef170a 100644 --- a/src/app-layer.h +++ b/src/app-layer.h @@ -63,7 +63,7 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *app_tctx, * * \param The internal protocol id. */ -AppProto AppLayerGetProtoByName(char *alproto_name); +AppProto AppLayerGetProtoByName(const char *alproto_name); /** * \brief Given the internal protocol id, returns a string representation From a686ab26dd23d6178aa54efa48dd8cc1d6ff5061 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 20 Jan 2025 09:05:04 +0100 Subject: [PATCH 08/17] detect: allow longer buffer names To support hook based buffer names. --- src/detect-engine.c | 2 +- src/detect.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/detect-engine.c b/src/detect-engine.c index 58de4691ed32..1979f709d671 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1020,7 +1020,7 @@ static void DetectBufferTypeFree(void) #endif static int DetectBufferTypeAdd(const char *string) { - BUG_ON(string == NULL || strlen(string) >= 32); + BUG_ON(string == NULL || strlen(string) >= 64); DetectBufferType *map = SCCalloc(1, sizeof(*map)); if (map == NULL) diff --git a/src/detect.h b/src/detect.h index ac7a0df3d764..f936e263ab6a 100644 --- a/src/detect.h +++ b/src/detect.h @@ -453,7 +453,7 @@ typedef struct DetectEngineAppInspectionEngine_ { } DetectEngineAppInspectionEngine; typedef struct DetectBufferType_ { - char name[32]; + char name[64]; char description[128]; int id; int parent_id; From 0ccc28cffe1f5e597600c54de05e5e03f277c522 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 17 Jan 2025 07:00:51 +0100 Subject: [PATCH 09/17] tls: fix handshake handling being too strict e.g. server hello done has no data --- src/app-layer-ssl.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index e387c6cc46c4..6ab097a7c045 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -1664,6 +1664,7 @@ static int SupportedHandshakeType(const uint8_t type) } /** + * \param input_len length of bytes after record header. Can be 0 (e.g. for server hello done). * \retval parsed number of consumed bytes * \retval < 0 error */ @@ -1673,9 +1674,6 @@ static int SSLv3ParseHandshakeType(SSLState *ssl_state, const uint8_t *input, const uint8_t *initial_input = input; int rc; - if (input_len == 0) { - return 0; - } DEBUG_VALIDATE_BUG_ON(RecordAlreadyProcessed(ssl_state->curr_connp)); switch (ssl_state->curr_connp->handshake_type) { From e391ce510c1abbf6fa49f932dec08efc8f9353d6 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 16 Jan 2025 12:32:40 +0100 Subject: [PATCH 10/17] tls: introduce per direction progress tracking Per direction track progress to be able to have more fine grained control over where the detection engines and logging hooks in. --- src/app-layer-ssl.c | 91 +++++++++++++++++++++---------- src/app-layer-ssl.h | 29 ++++++---- src/detect-tls-alpn.c | 8 +-- src/detect-tls-cert-fingerprint.c | 8 +-- src/detect-tls-cert-issuer.c | 8 +-- src/detect-tls-cert-serial.c | 8 +-- src/detect-tls-cert-subject.c | 8 +-- src/detect-tls-certs.c | 10 ++-- src/detect-tls-sni.c | 8 +-- src/detect-tls-subjectaltname.c | 2 +- src/detect-tls.c | 4 +- src/log-tlslog.c | 4 +- src/output-json-tls.c | 4 +- src/output-lua.c | 4 +- 14 files changed, 117 insertions(+), 79 deletions(-) diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index 6ab097a7c045..007d83cd6f1a 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -291,30 +291,36 @@ static uint64_t SSLGetTxCnt(void *state) return 1; } -static int SSLGetAlstateProgress(void *tx, uint8_t direction) +static void UpdateClientState(SSLState *ssl_state, enum TlsStateClient s) { - SSLState *ssl_state = (SSLState *)tx; - - /* we don't care about direction, only that app-layer parser is done - and have sent an EOF */ - if (ssl_state->flags & SSL_AL_FLAG_STATE_FINISHED) { - return TLS_STATE_FINISHED; - } +#ifdef DEBUG + enum TlsStateClient old = ssl_state->client_state; +#endif + ssl_state->client_state = s; +#ifdef DEBUG + SCLogDebug("toserver: state updated to %u from %u", s, old); +#endif +} - /* we want the logger to log when the handshake is done, even if the - state is not finished */ - if (ssl_state->flags & SSL_AL_FLAG_HANDSHAKE_DONE) { - return TLS_HANDSHAKE_DONE; - } +static void UpdateServerState(SSLState *ssl_state, enum TlsStateServer s) +{ +#ifdef DEBUG + enum TlsStateServer old = ssl_state->server_state; +#endif + ssl_state->server_state = s; +#ifdef DEBUG + SCLogDebug("toclient: state updated to %u from %u", s, old); +#endif +} - if (direction == STREAM_TOSERVER && - (ssl_state->server_connp.cert0_subject != NULL || - ssl_state->server_connp.cert0_issuerdn != NULL)) - { - return TLS_STATE_CERT_READY; +static int SSLGetAlstateProgress(void *tx, uint8_t direction) +{ + SSLState *ssl_state = (SSLState *)tx; + if (direction & STREAM_TOCLIENT) { + return ssl_state->server_state; + } else { + return ssl_state->client_state; } - - return TLS_STATE_IN_PROGRESS; } static AppLayerTxData *SSLGetTxData(void *vtx) @@ -1609,6 +1615,11 @@ static int TLSDecodeHandshakeHello(SSLState *ssl_state, ssl_state->curr_connp->ja3_hash = Ja3GenerateHash(ssl_state->curr_connp->ja3_str); } + if (ssl_state->curr_connp == &ssl_state->client_connp) { + UpdateClientState(ssl_state, TLS_STATE_CLIENT_HELLO_DONE); + } else { + UpdateServerState(ssl_state, TLS_STATE_SERVER_HELLO); + } end: return 0; } @@ -1635,6 +1646,11 @@ static inline int SSLv3ParseHandshakeTypeCertificate(SSLState *ssl_state, SSLSta SSLParserHSReset(connp); /* fall through to still consume the cert bytes */ } + if (connp == &ssl_state->client_connp) { + UpdateClientState(ssl_state, TLS_STATE_CLIENT_CERT_DONE); + } else { + UpdateServerState(ssl_state, TLS_STATE_SERVER_CERT_DONE); + } return input_len; } @@ -1703,7 +1719,6 @@ static int SSLv3ParseHandshakeType(SSLState *ssl_state, const uint8_t *input, break; case SSLV3_HS_CERTIFICATE: - rc = SSLv3ParseHandshakeTypeCertificate(ssl_state, direction ? &ssl_state->server_connp : &ssl_state->client_connp, initial_input, input_len); @@ -1727,6 +1742,9 @@ static int SSLv3ParseHandshakeType(SSLState *ssl_state, const uint8_t *input, SCLogDebug("new session ticket"); break; case SSLV3_HS_SERVER_HELLO_DONE: + if (direction) { + UpdateServerState(ssl_state, TLS_STATE_SERVER_HELLO_DONE); + } break; default: SSLSetEvent(ssl_state, TLS_DECODER_EVENT_INVALID_SSL_RECORD); @@ -2290,6 +2308,7 @@ static struct SSLDecoderResult SSLv2Decode(uint8_t direction, SSLState *ssl_stat ssl_state->current_flags = SSL_AL_FLAG_STATE_CLIENT_HELLO; ssl_state->current_flags |= SSL_AL_FLAG_SSL_CLIENT_HS; + UpdateClientState(ssl_state, TLS_STATE_CLIENT_HELLO_DONE); const uint16_t version = (uint16_t)(input[0] << 8) | input[1]; SCLogDebug("SSLv2: version %04x", version); @@ -2319,6 +2338,7 @@ static struct SSLDecoderResult SSLv2Decode(uint8_t direction, SSLState *ssl_stat } else { ssl_state->current_flags = SSL_AL_FLAG_STATE_CLIENT_KEYX; } + UpdateServerState(ssl_state, TLS_STATE_SERVER_CERT_DONE); /* fall through */ case SSLV2_MT_SERVER_VERIFY: @@ -2371,6 +2391,7 @@ static struct SSLDecoderResult SSLv2Decode(uint8_t direction, SSLState *ssl_stat case SSLV2_MT_SERVER_HELLO: ssl_state->current_flags = SSL_AL_FLAG_STATE_SERVER_HELLO; ssl_state->current_flags |= SSL_AL_FLAG_SSL_SERVER_HS; + UpdateServerState(ssl_state, TLS_STATE_SERVER_HELLO); break; } @@ -2556,7 +2577,11 @@ static struct SSLDecoderResult SSLv3Decode(uint8_t direction, SSLState *ssl_stat /* if we see (encrypted) application data, then this means the handshake must be done */ - ssl_state->flags |= SSL_AL_FLAG_HANDSHAKE_DONE; + if (ssl_state->curr_connp == &ssl_state->client_connp) { + UpdateClientState(ssl_state, TLS_STATE_CLIENT_HANDSHAKE_DONE); + } else { + UpdateServerState(ssl_state, TLS_STATE_SERVER_HANDSHAKE_DONE); + } if (ssl_config.encrypt_mode != SSL_CNF_ENC_HANDLE_FULL) { SCLogDebug("setting APP_LAYER_PARSER_NO_INSPECTION_PAYLOAD"); @@ -2684,7 +2709,10 @@ static AppLayerResult SSLDecode(Flow *f, uint8_t direction, void *alstate, (direction == 1 && AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC)))) { /* flag session as finished if APP_LAYER_PARSER_EOF is set */ - ssl_state->flags |= SSL_AL_FLAG_STATE_FINISHED; + if (direction == 0) + UpdateClientState(ssl_state, TLS_STATE_CLIENT_FINISHED); + else + UpdateServerState(ssl_state, TLS_STATE_SERVER_FINISHED); SCReturnStruct(APP_LAYER_OK); } else if (input == NULL || input_len == 0) { SCReturnStruct(APP_LAYER_ERROR); @@ -2792,19 +2820,22 @@ static AppLayerResult SSLDecode(Flow *f, uint8_t direction, void *alstate, /* mark handshake as done if we have subject and issuer */ if ((ssl_state->flags & SSL_AL_FLAG_NEED_CLIENT_CERT) && ssl_state->client_connp.cert0_subject && ssl_state->client_connp.cert0_issuerdn) { - SCLogDebug("SSL_AL_FLAG_HANDSHAKE_DONE"); - ssl_state->flags |= SSL_AL_FLAG_HANDSHAKE_DONE; + /* update both sides to keep existing behavior */ + UpdateClientState(ssl_state, TLS_STATE_CLIENT_HANDSHAKE_DONE); + UpdateServerState(ssl_state, TLS_STATE_SERVER_HANDSHAKE_DONE); } else if ((ssl_state->flags & SSL_AL_FLAG_NEED_CLIENT_CERT) == 0 && ssl_state->server_connp.cert0_subject && ssl_state->server_connp.cert0_issuerdn) { - SCLogDebug("SSL_AL_FLAG_HANDSHAKE_DONE"); - ssl_state->flags |= SSL_AL_FLAG_HANDSHAKE_DONE; + /* update both sides to keep existing behavior */ + UpdateClientState(ssl_state, TLS_STATE_CLIENT_HANDSHAKE_DONE); + UpdateServerState(ssl_state, TLS_STATE_SERVER_HANDSHAKE_DONE); } /* flag session as finished if APP_LAYER_PARSER_EOF is set */ if (AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TS) && AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC)) { - SCLogDebug("SSL_AL_FLAG_STATE_FINISHED"); - ssl_state->flags |= SSL_AL_FLAG_STATE_FINISHED; + /* update both sides to keep existing behavior */ + UpdateClientState(ssl_state, TLS_STATE_CLIENT_FINISHED); + UpdateServerState(ssl_state, TLS_STATE_SERVER_FINISHED); } return APP_LAYER_OK; @@ -3272,7 +3303,7 @@ void RegisterSSLParsers(void) AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_TLS, SSLGetAlstateProgress); AppLayerParserRegisterStateProgressCompletionStatus( - ALPROTO_TLS, TLS_STATE_FINISHED, TLS_STATE_FINISHED); + ALPROTO_TLS, TLS_STATE_CLIENT_FINISHED, TLS_STATE_SERVER_FINISHED); ConfNode *enc_handle = ConfGetNode("app-layer.protocols.tls.encryption-handling"); if (enc_handle != NULL && enc_handle->val != NULL) { diff --git a/src/app-layer-ssl.h b/src/app-layer-ssl.h index 479ccf51921d..484eb4cee98a 100644 --- a/src/app-layer-ssl.h +++ b/src/app-layer-ssl.h @@ -73,11 +73,21 @@ enum { TLS_DECODER_EVENT_INVALID_SSL_RECORD, }; -enum { - TLS_STATE_IN_PROGRESS = 0, - TLS_STATE_CERT_READY = 1, - TLS_HANDSHAKE_DONE = 2, - TLS_STATE_FINISHED = 3 +enum TlsStateClient { + TLS_STATE_CLIENT_IN_PROGRESS = 0, + TLS_STATE_CLIENT_HELLO_DONE, + TLS_STATE_CLIENT_CERT_DONE, + TLS_STATE_CLIENT_HANDSHAKE_DONE, + TLS_STATE_CLIENT_FINISHED, +}; + +enum TlsStateServer { + TLS_STATE_SERVER_IN_PROGRESS = 0, + TLS_STATE_SERVER_HELLO, + TLS_STATE_SERVER_CERT_DONE, + TLS_STATE_SERVER_HELLO_DONE, + TLS_STATE_SERVER_HANDSHAKE_DONE, + TLS_STATE_SERVER_FINISHED, }; /* Flag to indicate that server will now on send encrypted msgs */ @@ -101,17 +111,11 @@ enum { #define SSL_AL_FLAG_STATE_SERVER_KEYX BIT_U32(12) #define SSL_AL_FLAG_STATE_UNKNOWN BIT_U32(13) -/* flag to indicate that session is finished */ -#define SSL_AL_FLAG_STATE_FINISHED BIT_U32(14) - /* flags specific to HeartBeat state */ #define SSL_AL_FLAG_HB_INFLIGHT BIT_U32(15) #define SSL_AL_FLAG_HB_CLIENT_INIT BIT_U32(16) #define SSL_AL_FLAG_HB_SERVER_INIT BIT_U32(17) -/* flag to indicate that handshake is done */ -#define SSL_AL_FLAG_HANDSHAKE_DONE BIT_U32(18) - /* Session resumed without a full handshake */ #define SSL_AL_FLAG_SESSION_RESUMED BIT_U32(20) @@ -311,6 +315,9 @@ typedef struct SSLState_ { SSLStateConnp *curr_connp; + enum TlsStateClient client_state; + enum TlsStateServer server_state; + SSLStateConnp client_connp; SSLStateConnp server_connp; } SSLState; diff --git a/src/detect-tls-alpn.c b/src/detect-tls-alpn.c index b4aa82f9c52a..4926b6cb7b5d 100644 --- a/src/detect-tls-alpn.c +++ b/src/detect-tls-alpn.c @@ -70,10 +70,10 @@ void DetectTlsAlpnRegister(void) sigmatch_table[DETECT_TLS_ALPN].flags |= SIGMATCH_NOOPT; sigmatch_table[DETECT_TLS_ALPN].flags |= SIGMATCH_INFO_STICKY_BUFFER; - DetectAppLayerMultiRegister("tls.alpn", ALPROTO_TLS, SIG_FLAG_TOSERVER, 0, TlsAlpnGetData, 2, - TLS_STATE_IN_PROGRESS); - DetectAppLayerMultiRegister( - "tls.alpn", ALPROTO_TLS, SIG_FLAG_TOCLIENT, 0, TlsAlpnGetData, 2, TLS_STATE_CERT_READY); + DetectAppLayerMultiRegister("tls.alpn", ALPROTO_TLS, SIG_FLAG_TOSERVER, + TLS_STATE_CLIENT_HELLO_DONE, TlsAlpnGetData, 2, TLS_STATE_CLIENT_HELLO_DONE); + DetectAppLayerMultiRegister("tls.alpn", ALPROTO_TLS, SIG_FLAG_TOCLIENT, TLS_STATE_SERVER_HELLO, + TlsAlpnGetData, 2, TLS_STATE_SERVER_HELLO); DetectBufferTypeSetDescriptionByName("tls.alpn", "TLS APLN"); diff --git a/src/detect-tls-cert-fingerprint.c b/src/detect-tls-cert-fingerprint.c index 8f6872a81871..8b1dead7cc98 100644 --- a/src/detect-tls-cert-fingerprint.c +++ b/src/detect-tls-cert-fingerprint.c @@ -85,16 +85,16 @@ void DetectTlsFingerprintRegister(void) sigmatch_table[DETECT_TLS_CERT_FINGERPRINT].flags |= SIGMATCH_INFO_STICKY_BUFFER; DetectAppLayerInspectEngineRegister("tls.cert_fingerprint", ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_fingerprint", SIG_FLAG_TOCLIENT, 2, - PrefilterGenericMpmRegister, GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + PrefilterGenericMpmRegister, GetData, ALPROTO_TLS, TLS_STATE_SERVER_CERT_DONE); DetectAppLayerInspectEngineRegister("tls.cert_fingerprint", ALPROTO_TLS, SIG_FLAG_TOSERVER, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_CLIENT_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_fingerprint", SIG_FLAG_TOSERVER, 2, - PrefilterGenericMpmRegister, GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + PrefilterGenericMpmRegister, GetData, ALPROTO_TLS, TLS_STATE_CLIENT_CERT_DONE); DetectBufferTypeSetDescriptionByName("tls.cert_fingerprint", "TLS certificate fingerprint"); diff --git a/src/detect-tls-cert-issuer.c b/src/detect-tls-cert-issuer.c index 55411172a651..04a4b5773f07 100644 --- a/src/detect-tls-cert-issuer.c +++ b/src/detect-tls-cert-issuer.c @@ -80,16 +80,16 @@ void DetectTlsIssuerRegister(void) sigmatch_table[DETECT_TLS_CERT_ISSUER].flags |= SIGMATCH_INFO_STICKY_BUFFER; DetectAppLayerInspectEngineRegister("tls.cert_issuer", ALPROTO_TLS, SIG_FLAG_TOSERVER, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_CLIENT_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_issuer", SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, - GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + GetData, ALPROTO_TLS, TLS_STATE_CLIENT_CERT_DONE); DetectAppLayerInspectEngineRegister("tls.cert_issuer", ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_issuer", SIG_FLAG_TOCLIENT, 2, PrefilterGenericMpmRegister, - GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + GetData, ALPROTO_TLS, TLS_STATE_SERVER_CERT_DONE); DetectBufferTypeSetDescriptionByName("tls.cert_issuer", "TLS certificate issuer"); diff --git a/src/detect-tls-cert-serial.c b/src/detect-tls-cert-serial.c index fb4e02995433..ece7eebaa3f8 100644 --- a/src/detect-tls-cert-serial.c +++ b/src/detect-tls-cert-serial.c @@ -84,16 +84,16 @@ void DetectTlsSerialRegister(void) sigmatch_table[DETECT_TLS_CERT_SERIAL].flags |= SIGMATCH_INFO_STICKY_BUFFER; DetectAppLayerInspectEngineRegister("tls.cert_serial", ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_serial", SIG_FLAG_TOCLIENT, 2, PrefilterGenericMpmRegister, - GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + GetData, ALPROTO_TLS, TLS_STATE_SERVER_CERT_DONE); DetectAppLayerInspectEngineRegister("tls.cert_serial", ALPROTO_TLS, SIG_FLAG_TOSERVER, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_CLIENT_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_serial", SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, - GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + GetData, ALPROTO_TLS, TLS_STATE_CLIENT_CERT_DONE); DetectBufferTypeSetDescriptionByName("tls.cert_serial", "TLS certificate serial number"); diff --git a/src/detect-tls-cert-subject.c b/src/detect-tls-cert-subject.c index 7d3ca50a6190..83f5649ec2c7 100644 --- a/src/detect-tls-cert-subject.c +++ b/src/detect-tls-cert-subject.c @@ -80,16 +80,16 @@ void DetectTlsSubjectRegister(void) sigmatch_table[DETECT_TLS_CERT_SUBJECT].flags |= SIGMATCH_INFO_STICKY_BUFFER; DetectAppLayerInspectEngineRegister("tls.cert_subject", ALPROTO_TLS, SIG_FLAG_TOSERVER, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_CLIENT_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_subject", SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, - GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + GetData, ALPROTO_TLS, TLS_STATE_CLIENT_CERT_DONE); DetectAppLayerInspectEngineRegister("tls.cert_subject", ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectBufferGeneric, GetData); + TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectBufferGeneric, GetData); DetectAppLayerMpmRegister("tls.cert_subject", SIG_FLAG_TOCLIENT, 2, PrefilterGenericMpmRegister, - GetData, ALPROTO_TLS, TLS_STATE_CERT_READY); + GetData, ALPROTO_TLS, TLS_STATE_SERVER_CERT_DONE); DetectBufferTypeSupportsMultiInstance("tls.cert_subject"); diff --git a/src/detect-tls-certs.c b/src/detect-tls-certs.c index a626769994ed..1f2ef564fc69 100644 --- a/src/detect-tls-certs.c +++ b/src/detect-tls-certs.c @@ -122,10 +122,10 @@ void DetectTlsCertsRegister(void) sigmatch_table[DETECT_TLS_CERTS].flags |= SIGMATCH_NOOPT; sigmatch_table[DETECT_TLS_CERTS].flags |= SIGMATCH_INFO_STICKY_BUFFER; - DetectAppLayerMultiRegister("tls.certs", ALPROTO_TLS, SIG_FLAG_TOCLIENT, TLS_STATE_CERT_READY, - TlsCertsGetData, 2, 1); - DetectAppLayerMultiRegister("tls.certs", ALPROTO_TLS, SIG_FLAG_TOSERVER, TLS_STATE_CERT_READY, - TlsCertsGetData, 2, 1); + DetectAppLayerMultiRegister("tls.certs", ALPROTO_TLS, SIG_FLAG_TOCLIENT, + TLS_STATE_SERVER_CERT_DONE, TlsCertsGetData, 2, 1); + DetectAppLayerMultiRegister("tls.certs", ALPROTO_TLS, SIG_FLAG_TOSERVER, + TLS_STATE_CLIENT_CERT_DONE, TlsCertsGetData, 2, 1); DetectBufferTypeSetDescriptionByName("tls.certs", "TLS certificate"); @@ -253,7 +253,7 @@ void DetectTlsCertChainLenRegister(void) sigmatch_table[KEYWORD_ID].Free = DetectTLSCertChainLenFree; DetectAppLayerInspectEngineRegister(BUFFER_NAME, ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectGenericList, NULL); + TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectGenericList, NULL); g_tls_cert_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME); } diff --git a/src/detect-tls-sni.c b/src/detect-tls-sni.c index 1ec7400b3437..8e0a3c18fd63 100644 --- a/src/detect-tls-sni.c +++ b/src/detect-tls-sni.c @@ -73,11 +73,11 @@ void DetectTlsSniRegister(void) sigmatch_table[DETECT_TLS_SNI].flags |= SIGMATCH_NOOPT; sigmatch_table[DETECT_TLS_SNI].flags |= SIGMATCH_INFO_STICKY_BUFFER; - DetectAppLayerInspectEngineRegister("tls.sni", ALPROTO_TLS, SIG_FLAG_TOSERVER, 0, - DetectEngineInspectBufferGeneric, GetData); + DetectAppLayerInspectEngineRegister("tls.sni", ALPROTO_TLS, SIG_FLAG_TOSERVER, + TLS_STATE_CLIENT_HELLO_DONE, DetectEngineInspectBufferGeneric, GetData); - DetectAppLayerMpmRegister( - "tls.sni", SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, GetData, ALPROTO_TLS, 0); + DetectAppLayerMpmRegister("tls.sni", SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, GetData, + ALPROTO_TLS, TLS_STATE_CLIENT_HELLO_DONE); DetectBufferTypeSetDescriptionByName("tls.sni", "TLS Server Name Indication (SNI) extension"); diff --git a/src/detect-tls-subjectaltname.c b/src/detect-tls-subjectaltname.c index f07b54dc3c72..fa431e03855b 100644 --- a/src/detect-tls-subjectaltname.c +++ b/src/detect-tls-subjectaltname.c @@ -72,7 +72,7 @@ void DetectTlsSubjectAltNameRegister(void) sigmatch_table[DETECT_TLS_SUBJECTALTNAME].flags |= SIGMATCH_INFO_STICKY_BUFFER; DetectAppLayerMultiRegister("tls.subjectaltname", ALPROTO_TLS, SIG_FLAG_TOCLIENT, 0, - TlsSubjectAltNameGetData, 2, TLS_STATE_CERT_READY); + TlsSubjectAltNameGetData, 2, TLS_STATE_SERVER_CERT_DONE); DetectBufferTypeSetDescriptionByName("tls.subjectaltname", "TLS Subject Alternative Name"); diff --git a/src/detect-tls.c b/src/detect-tls.c index a0e423dbe01b..b16c3db30302 100644 --- a/src/detect-tls.c +++ b/src/detect-tls.c @@ -144,10 +144,10 @@ void DetectTlsRegister (void) g_tls_cert_fingerprint_list_id = DetectBufferTypeRegister("tls.cert_fingerprint"); DetectAppLayerInspectEngineRegister("tls_cert", ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_CERT_READY, DetectEngineInspectGenericList, NULL); + TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectGenericList, NULL); DetectAppLayerInspectEngineRegister("tls_cert", ALPROTO_TLS, SIG_FLAG_TOSERVER, - TLS_STATE_CERT_READY, DetectEngineInspectGenericList, NULL); + TLS_STATE_CLIENT_CERT_DONE, DetectEngineInspectGenericList, NULL); } /** diff --git a/src/log-tlslog.c b/src/log-tlslog.c index 8d88047e9931..a39c07f15d26 100644 --- a/src/log-tlslog.c +++ b/src/log-tlslog.c @@ -500,6 +500,6 @@ static int LogTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, void LogTlsLogRegister(void) { OutputRegisterTxModuleWithProgress(LOGGER_TLS, MODULE_NAME, "tls-log", LogTlsLogInitCtx, - ALPROTO_TLS, LogTlsLogger, TLS_HANDSHAKE_DONE, TLS_HANDSHAKE_DONE, LogTlsLogThreadInit, - LogTlsLogThreadDeinit); + ALPROTO_TLS, LogTlsLogger, TLS_STATE_CLIENT_HANDSHAKE_DONE, + TLS_STATE_SERVER_HANDSHAKE_DONE, LogTlsLogThreadInit, LogTlsLogThreadDeinit); } diff --git a/src/output-json-tls.c b/src/output-json-tls.c index c4ba0e249e62..2fdb321baaea 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -669,6 +669,6 @@ void JsonTlsLogRegister (void) { /* register as child of eve-log */ OutputRegisterTxSubModuleWithProgress(LOGGER_JSON_TX, "eve-log", "JsonTlsLog", "eve-log.tls", - OutputTlsLogInitSub, ALPROTO_TLS, JsonTlsLogger, TLS_HANDSHAKE_DONE, TLS_HANDSHAKE_DONE, - JsonTlsLogThreadInit, JsonTlsLogThreadDeinit); + OutputTlsLogInitSub, ALPROTO_TLS, JsonTlsLogger, TLS_STATE_SERVER_HANDSHAKE_DONE, + TLS_STATE_CLIENT_HANDSHAKE_DONE, JsonTlsLogThreadInit, JsonTlsLogThreadDeinit); } diff --git a/src/output-lua.c b/src/output-lua.c index f2461eb24321..fe080228943d 100644 --- a/src/output-lua.c +++ b/src/output-lua.c @@ -791,8 +791,8 @@ static OutputInitResult OutputLuaLogInit(ConfNode *conf) } else if (opts.alproto == ALPROTO_TLS) { om->TxLogFunc = LuaTxLogger; om->alproto = ALPROTO_TLS; - om->tc_log_progress = TLS_HANDSHAKE_DONE; - om->ts_log_progress = TLS_HANDSHAKE_DONE; + om->tc_log_progress = TLS_STATE_SERVER_HANDSHAKE_DONE; + om->ts_log_progress = TLS_STATE_CLIENT_HANDSHAKE_DONE; AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_TLS); } else if (opts.alproto == ALPROTO_DNS) { om->TxLogFunc = LuaTxLogger; From c12cc12e4be6dea949965d25932fa2de05910c5e Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 17 Jan 2025 12:10:01 +0100 Subject: [PATCH 11/17] app-layer: API for mapping progress name vs id --- rust/src/applayer.rs | 5 +++ rust/src/applayertemplate/template.rs | 2 ++ rust/src/bittorrent_dht/bittorrent_dht.rs | 2 ++ rust/src/dcerpc/dcerpc.rs | 2 ++ rust/src/dcerpc/dcerpc_udp.rs | 2 ++ rust/src/dhcp/dhcp.rs | 2 ++ rust/src/dns/dns.rs | 4 +++ rust/src/enip/enip.rs | 2 ++ rust/src/http2/http2.rs | 2 ++ rust/src/ike/ike.rs | 2 ++ rust/src/krb/krb5.rs | 2 ++ rust/src/ldap/ldap.rs | 4 +++ rust/src/modbus/modbus.rs | 2 ++ rust/src/mqtt/mqtt.rs | 2 ++ rust/src/nfs/nfs.rs | 4 +++ rust/src/ntp/ntp.rs | 2 ++ rust/src/pgsql/pgsql.rs | 2 ++ rust/src/quic/quic.rs | 2 ++ rust/src/rdp/rdp.rs | 2 ++ rust/src/rfb/rfb.rs | 2 ++ rust/src/sip/sip.rs | 2 ++ rust/src/smb/smb.rs | 2 ++ rust/src/snmp/snmp.rs | 2 ++ rust/src/ssh/ssh.rs | 2 ++ rust/src/telnet/telnet.rs | 2 ++ rust/src/websocket/websocket.rs | 2 ++ src/app-layer-parser.c | 42 +++++++++++++++++++++++ src/app-layer-parser.h | 26 ++++++++++++++ src/app-layer-register.c | 5 +++ src/app-layer-register.h | 2 ++ 30 files changed, 136 insertions(+) diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index 12bc3d42bfef..d6be720d7c04 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -410,6 +410,9 @@ pub struct RustParser { pub get_frame_id_by_name: Option, pub get_frame_name_by_id: Option, + + pub get_state_id_by_name: Option, + pub get_state_name_by_id: Option, } /// Create a slice, given a buffer and a length @@ -470,6 +473,8 @@ pub type GetStateDataFn = unsafe extern "C" fn(*mut c_void) -> *mut AppLayerStat pub type ApplyTxConfigFn = unsafe extern "C" fn (*mut c_void, *mut c_void, c_int, AppLayerTxConfig); pub type GetFrameIdByName = unsafe extern "C" fn(*const c_char) -> c_int; pub type GetFrameNameById = unsafe extern "C" fn(u8) -> *const c_char; +pub type GetStateIdByName = unsafe extern "C" fn(*const c_char, u8) -> c_int; +pub type GetStateNameById = unsafe extern "C" fn(c_int, u8) -> *const c_char; // Defined in app-layer-register.h diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs index 9f706a74bba0..2b141fd3e0e6 100644 --- a/rust/src/applayertemplate/template.rs +++ b/rust/src/applayertemplate/template.rs @@ -396,6 +396,8 @@ pub unsafe extern "C" fn rs_template_register_parser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/bittorrent_dht/bittorrent_dht.rs b/rust/src/bittorrent_dht/bittorrent_dht.rs index 812625b3debd..dcb01ea01d62 100644 --- a/rust/src/bittorrent_dht/bittorrent_dht.rs +++ b/rust/src/bittorrent_dht/bittorrent_dht.rs @@ -286,6 +286,8 @@ pub unsafe extern "C" fn rs_bittorrent_dht_udp_register_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/dcerpc/dcerpc.rs b/rust/src/dcerpc/dcerpc.rs index 57a57e6e1f8d..352d2b8b45ec 100644 --- a/rust/src/dcerpc/dcerpc.rs +++ b/rust/src/dcerpc/dcerpc.rs @@ -1282,6 +1282,8 @@ pub unsafe extern "C" fn rs_dcerpc_register_parser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: Some(DCERPCFrameType::ffi_id_from_name), get_frame_name_by_id: Some(DCERPCFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/dcerpc/dcerpc_udp.rs b/rust/src/dcerpc/dcerpc_udp.rs index 634e02ad6056..871e9394dc80 100644 --- a/rust/src/dcerpc/dcerpc_udp.rs +++ b/rust/src/dcerpc/dcerpc_udp.rs @@ -384,6 +384,8 @@ pub unsafe extern "C" fn rs_dcerpc_udp_register_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/dhcp/dhcp.rs b/rust/src/dhcp/dhcp.rs index 965535375021..620fe2bf3af1 100644 --- a/rust/src/dhcp/dhcp.rs +++ b/rust/src/dhcp/dhcp.rs @@ -288,6 +288,8 @@ pub unsafe extern "C" fn SCRegisterDhcpParser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/dns/dns.rs b/rust/src/dns/dns.rs index 30c1254274b8..3d048200ba53 100644 --- a/rust/src/dns/dns.rs +++ b/rust/src/dns/dns.rs @@ -1122,6 +1122,8 @@ pub unsafe extern "C" fn SCRegisterDnsUdpParser() { flags: 0, get_frame_id_by_name: Some(DnsFrameType::ffi_id_from_name), get_frame_name_by_id: Some(DnsFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); @@ -1167,6 +1169,8 @@ pub unsafe extern "C" fn SCRegisterDnsTcpParser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: Some(DnsFrameType::ffi_id_from_name), get_frame_name_by_id: Some(DnsFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/enip/enip.rs b/rust/src/enip/enip.rs index d58efde5b00f..95cb021eae5c 100644 --- a/rust/src/enip/enip.rs +++ b/rust/src/enip/enip.rs @@ -619,6 +619,8 @@ pub unsafe extern "C" fn SCEnipRegisterParsers() { flags: 0, get_frame_id_by_name: Some(EnipFrameType::ffi_id_from_name), get_frame_name_by_id: Some(EnipFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index fd5cf12abd31..6d9f35ceae6f 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -1570,6 +1570,8 @@ pub unsafe extern "C" fn rs_http2_register_parser() { flags: 0, get_frame_id_by_name: Some(Http2FrameType::ffi_id_from_name), get_frame_name_by_id: Some(Http2FrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/ike/ike.rs b/rust/src/ike/ike.rs index 297b01f61c49..1c9ee89f5b8c 100644 --- a/rust/src/ike/ike.rs +++ b/rust/src/ike/ike.rs @@ -428,6 +428,8 @@ pub unsafe extern "C" fn rs_ike_register_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/krb/krb5.rs b/rust/src/krb/krb5.rs index 9d90cdd992a7..abe146fee517 100644 --- a/rust/src/krb/krb5.rs +++ b/rust/src/krb/krb5.rs @@ -613,6 +613,8 @@ pub unsafe extern "C" fn rs_register_krb5_parser() { flags : 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; // register UDP parser let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/ldap/ldap.rs b/rust/src/ldap/ldap.rs index 7c0ce1960b94..8e174ef69cf2 100644 --- a/rust/src/ldap/ldap.rs +++ b/rust/src/ldap/ldap.rs @@ -678,6 +678,8 @@ pub unsafe extern "C" fn SCRegisterLdapTcpParser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: Some(LdapFrameType::ffi_id_from_name), get_frame_name_by_id: Some(LdapFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); @@ -735,6 +737,8 @@ pub unsafe extern "C" fn SCRegisterLdapUdpParser() { flags: 0, get_frame_id_by_name: Some(LdapFrameType::ffi_id_from_name), get_frame_name_by_id: Some(LdapFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/modbus/modbus.rs b/rust/src/modbus/modbus.rs index 33cb9787c47e..f2f6cf5d0015 100644 --- a/rust/src/modbus/modbus.rs +++ b/rust/src/modbus/modbus.rs @@ -418,6 +418,8 @@ pub unsafe extern "C" fn rs_modbus_register_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/mqtt/mqtt.rs b/rust/src/mqtt/mqtt.rs index 697d2ef266cb..4d8cc5cdc4b3 100644 --- a/rust/src/mqtt/mqtt.rs +++ b/rust/src/mqtt/mqtt.rs @@ -797,6 +797,8 @@ pub unsafe extern "C" fn SCMqttRegisterParser() { flags: 0, get_frame_id_by_name: Some(MQTTFrameType::ffi_id_from_name), get_frame_name_by_id: Some(MQTTFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index 48a6bba1c18b..3ad5e5f69754 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -2000,6 +2000,8 @@ pub unsafe extern "C" fn rs_nfs_register_parser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: Some(NFSFrameType::ffi_id_from_name), get_frame_name_by_id: Some(NFSFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); @@ -2077,6 +2079,8 @@ pub unsafe extern "C" fn rs_nfs_udp_register_parser() { flags: 0, get_frame_id_by_name: Some(NFSFrameType::ffi_id_from_name), get_frame_name_by_id: Some(NFSFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/ntp/ntp.rs b/rust/src/ntp/ntp.rs index 576e1b6fc80d..f233a8767b1c 100644 --- a/rust/src/ntp/ntp.rs +++ b/rust/src/ntp/ntp.rs @@ -285,6 +285,8 @@ pub unsafe extern "C" fn rs_register_ntp_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/pgsql/pgsql.rs b/rust/src/pgsql/pgsql.rs index 4054eac0c3c7..9b73f3e810b6 100644 --- a/rust/src/pgsql/pgsql.rs +++ b/rust/src/pgsql/pgsql.rs @@ -811,6 +811,8 @@ pub unsafe extern "C" fn SCRegisterPgsqlParser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/quic/quic.rs b/rust/src/quic/quic.rs index 442cfc944a65..593540b6c296 100644 --- a/rust/src/quic/quic.rs +++ b/rust/src/quic/quic.rs @@ -510,6 +510,8 @@ pub unsafe extern "C" fn rs_quic_register_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/rdp/rdp.rs b/rust/src/rdp/rdp.rs index b5684aa248a9..e7f1c257845a 100644 --- a/rust/src/rdp/rdp.rs +++ b/rust/src/rdp/rdp.rs @@ -496,6 +496,8 @@ pub unsafe extern "C" fn rs_rdp_register_parser() { flags: 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = std::ffi::CString::new("tcp").unwrap(); diff --git a/rust/src/rfb/rfb.rs b/rust/src/rfb/rfb.rs index 6901c973dd74..41231537d2db 100644 --- a/rust/src/rfb/rfb.rs +++ b/rust/src/rfb/rfb.rs @@ -871,6 +871,8 @@ pub unsafe extern "C" fn SCRfbRegisterParser() { flags: 0, get_frame_id_by_name: Some(RFBFrameType::ffi_id_from_name), get_frame_name_by_id: Some(RFBFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/sip/sip.rs b/rust/src/sip/sip.rs index 1743953f1f33..afb1125def98 100755 --- a/rust/src/sip/sip.rs +++ b/rust/src/sip/sip.rs @@ -583,6 +583,8 @@ pub unsafe extern "C" fn rs_sip_register_parser() { flags: 0, get_frame_id_by_name: Some(SIPFrameType::ffi_id_from_name), get_frame_name_by_id: Some(SIPFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs index 43a44545ad09..837aa4c7504e 100644 --- a/rust/src/smb/smb.rs +++ b/rust/src/smb/smb.rs @@ -2374,6 +2374,8 @@ pub unsafe extern "C" fn rs_smb_register_parser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: Some(SMBFrameType::ffi_id_from_name), get_frame_name_by_id: Some(SMBFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/snmp/snmp.rs b/rust/src/snmp/snmp.rs index a30e182aefb4..fc621bca1b50 100644 --- a/rust/src/snmp/snmp.rs +++ b/rust/src/snmp/snmp.rs @@ -410,6 +410,8 @@ pub unsafe extern "C" fn rs_register_snmp_parser() { flags : 0, get_frame_id_by_name: None, get_frame_name_by_id: None, + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("udp").unwrap(); if AppLayerProtoDetectConfProtoDetectionEnabled(ip_proto_str.as_ptr(), parser.name) != 0 { diff --git a/rust/src/ssh/ssh.rs b/rust/src/ssh/ssh.rs index e0c4ecc7c0c8..56c7c40116d0 100644 --- a/rust/src/ssh/ssh.rs +++ b/rust/src/ssh/ssh.rs @@ -525,6 +525,8 @@ pub unsafe extern "C" fn SCRegisterSshParser() { flags: 0, get_frame_id_by_name: Some(SshFrameType::ffi_id_from_name), get_frame_name_by_id: Some(SshFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/rust/src/telnet/telnet.rs b/rust/src/telnet/telnet.rs index 01f0e38c9ab4..498fe34c0b5c 100644 --- a/rust/src/telnet/telnet.rs +++ b/rust/src/telnet/telnet.rs @@ -540,6 +540,8 @@ pub unsafe extern "C" fn rs_telnet_register_parser() { flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS, get_frame_id_by_name: Some(TelnetFrameType::ffi_id_from_name), get_frame_name_by_id: Some(TelnetFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; diff --git a/rust/src/websocket/websocket.rs b/rust/src/websocket/websocket.rs index 3f49efbc19ee..cfcb7f687704 100644 --- a/rust/src/websocket/websocket.rs +++ b/rust/src/websocket/websocket.rs @@ -369,6 +369,8 @@ pub unsafe extern "C" fn rs_websocket_register_parser() { flags: 0, // do not accept gaps as there is no good way to resync get_frame_id_by_name: Some(WebSocketFrameType::ffi_id_from_name), get_frame_name_by_id: Some(WebSocketFrameType::ffi_name_from_id), + get_state_id_by_name: None, + get_state_name_by_id: None, }; let ip_proto_str = CString::new("tcp").unwrap(); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index ec263aeb7859..856980fa0353 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -107,6 +107,9 @@ typedef struct AppLayerParserProtoCtx_ AppLayerParserGetFrameIdByNameFn GetFrameIdByName; AppLayerParserGetFrameNameByIdFn GetFrameNameById; + AppLayerParserGetStateIdByNameFn GetStateIdByName; + AppLayerParserGetStateNameByIdFn GetStateNameById; + /* each app-layer has its own value */ uint32_t stream_depth; @@ -547,6 +550,16 @@ void AppLayerParserRegisterGetEventInfoById(uint8_t ipproto, AppProto alproto, SCReturn; } +void AppLayerParserRegisterGetStateFuncs(uint8_t ipproto, AppProto alproto, + AppLayerParserGetStateIdByNameFn GetIdByNameFunc, + AppLayerParserGetStateNameByIdFn GetNameByIdFunc) +{ + SCEnter(); + alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetStateIdByName = GetIdByNameFunc; + alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetStateNameById = GetNameByIdFunc; + SCReturn; +} + void AppLayerParserRegisterGetFrameFuncs(uint8_t ipproto, AppProto alproto, AppLayerParserGetFrameIdByNameFn GetIdByNameFunc, AppLayerParserGetFrameNameByIdFn GetNameByIdFunc) @@ -1578,6 +1591,35 @@ void AppLayerParserSetStreamDepthFlag(uint8_t ipproto, AppProto alproto, void *s SCReturn; } +/** + * \param id progress value id to get the name for + * \param direction STREAM_TOSERVER/STREAM_TOCLIENT + */ +int AppLayerParserGetStateIdByName( + uint8_t ipproto, AppProto alproto, const char *name, const uint8_t direction) +{ + if (alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetStateIdByName != NULL) { + return alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetStateIdByName( + name, direction); + } else { + return -1; + } +} + +/** + * \param id progress value id to get the name for + * \param direction STREAM_TOSERVER/STREAM_TOCLIENT + */ +const char *AppLayerParserGetStateNameById( + uint8_t ipproto, AppProto alproto, const int id, const uint8_t direction) +{ + if (alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetStateNameById != NULL) { + return alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetStateNameById(id, direction); + } else { + return NULL; + } +} + int AppLayerParserGetFrameIdByName(uint8_t ipproto, AppProto alproto, const char *name) { if (alp_ctx.ctxs[alproto][FlowGetProtoMapping(ipproto)].GetFrameIdByName != NULL) { diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index d233edf9eb1f..c9de59bb779a 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -155,6 +155,17 @@ typedef AppLayerGetTxIterTuple (*AppLayerGetTxIteratorFunc) /***** Parser related registration *****/ +/** + * \param name progress name to get the id for + * \param direction STREAM_TOSERVER/STREAM_TOCLIENT + */ +typedef int (*AppLayerParserGetStateIdByNameFn)(const char *name, const uint8_t direction); +/** + * \param id progress value id to get the name for + * \param direction STREAM_TOSERVER/STREAM_TOCLIENT + */ +typedef const char *(*AppLayerParserGetStateNameByIdFn)(const int id, const uint8_t direction); + typedef int (*AppLayerParserGetFrameIdByNameFn)(const char *frame_name); typedef const char *(*AppLayerParserGetFrameNameByIdFn)(const uint8_t id); @@ -206,6 +217,9 @@ void AppLayerParserRegisterGetFrameFuncs(uint8_t ipproto, AppProto alproto, AppLayerParserGetFrameNameByIdFn GetFrameNameById); void AppLayerParserRegisterSetStreamDepthFlag(uint8_t ipproto, AppProto alproto, void (*SetStreamDepthFlag)(void *tx, uint8_t flags)); +void AppLayerParserRegisterGetStateFuncs(uint8_t ipproto, AppProto alproto, + AppLayerParserGetStateIdByNameFn GetStateIdByName, + AppLayerParserGetStateNameByIdFn GetStateNameById); void AppLayerParserRegisterTxDataFunc(uint8_t ipproto, AppProto alproto, AppLayerTxData *(*GetTxData)(void *tx)); @@ -293,6 +307,18 @@ void AppLayerParserSetStreamDepthFlag(uint8_t ipproto, AppProto alproto, void *s int AppLayerParserIsEnabled(AppProto alproto); int AppLayerParserGetFrameIdByName(uint8_t ipproto, AppProto alproto, const char *name); const char *AppLayerParserGetFrameNameById(uint8_t ipproto, AppProto alproto, const uint8_t id); +/** + * \param name progress name to get the id for + * \param direction STREAM_TOSERVER/STREAM_TOCLIENT + */ +int AppLayerParserGetStateIdByName( + uint8_t ipproto, AppProto alproto, const char *name, uint8_t direction); +/** + * \param id progress value id to get the name for + * \param direction STREAM_TOSERVER/STREAM_TOCLIENT + */ +const char *AppLayerParserGetStateNameById( + uint8_t ipproto, AppProto alproto, const int id, uint8_t direction); /***** Cleanup *****/ diff --git a/src/app-layer-register.c b/src/app-layer-register.c index 9a33a38eea76..109804dc1e18 100644 --- a/src/app-layer-register.c +++ b/src/app-layer-register.c @@ -188,6 +188,11 @@ int AppLayerRegisterParser(const struct AppLayerParser *p, AppProto alproto) p->ip_proto, alproto, p->GetFrameIdByName, p->GetFrameNameById); } + if (p->GetStateIdByName && p->GetStateNameById) { + AppLayerParserRegisterGetStateFuncs( + p->ip_proto, alproto, p->GetStateIdByName, p->GetStateNameById); + } + return 0; } diff --git a/src/app-layer-register.h b/src/app-layer-register.h index 6f489c73e248..a6528b76b1ae 100644 --- a/src/app-layer-register.h +++ b/src/app-layer-register.h @@ -74,6 +74,8 @@ typedef struct AppLayerParser { AppLayerParserGetFrameIdByNameFn GetFrameIdByName; AppLayerParserGetFrameNameByIdFn GetFrameNameById; + AppLayerParserGetStateIdByNameFn GetStateIdByName; + AppLayerParserGetStateNameByIdFn GetStateNameById; } AppLayerParser; /** From 68622575ce0e9a5a99365d3c7d3cb9af7e8b9382 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 17 Jan 2025 12:10:29 +0100 Subject: [PATCH 12/17] tls: expose progress by name --- src/app-layer-ssl.c | 75 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index 007d83cd6f1a..0aec8b90acf3 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -43,6 +43,58 @@ #include "util-enum.h" #include "util-validate.h" +static SCEnumCharMap tls_state_client_table[] = { + { + "client_in_progress", + TLS_STATE_CLIENT_IN_PROGRESS, + }, + { + "client_hello_done", + TLS_STATE_CLIENT_HELLO_DONE, + }, + { + "client_cert_done", + TLS_STATE_CLIENT_CERT_DONE, + }, + { + "client_handshake_done", + TLS_STATE_CLIENT_HANDSHAKE_DONE, + }, + { + "client_finished", + TLS_STATE_CLIENT_FINISHED, + }, + { NULL, -1 }, +}; + +static SCEnumCharMap tls_state_server_table[] = { + { + "server_in_progress", + TLS_STATE_SERVER_IN_PROGRESS, + }, + { + "server_hello", + TLS_STATE_SERVER_HELLO, + }, + { + "server_cert_done", + TLS_STATE_SERVER_CERT_DONE, + }, + { + "server_hello_done", + TLS_STATE_SERVER_HELLO_DONE, + }, + { + "server_handshake_done", + TLS_STATE_SERVER_HANDSHAKE_DONE, + }, + { + "server_finished", + TLS_STATE_SERVER_FINISHED, + }, + { NULL, -1 }, +}; + SCEnumCharMap tls_frame_table[] = { { "pdu", @@ -2996,6 +3048,26 @@ static AppProto SSLProbingParser(Flow *f, uint8_t direction, return ALPROTO_FAILED; } +static int SSLStateGetStateIdByName(const char *name, const uint8_t direction) +{ + SCEnumCharMap *map = + direction == STREAM_TOSERVER ? tls_state_client_table : tls_state_server_table; + + int id = SCMapEnumNameToValue(name, map); + if (id < 0) { + return -1; + } + return id; +} + +static const char *SSLStateGetStateNameById(const int id, const uint8_t direction) +{ + SCEnumCharMap *map = + direction == STREAM_TOSERVER ? tls_state_client_table : tls_state_server_table; + const char *name = SCMapEnumValueToName(id, map); + return name; +} + static int SSLStateGetFrameIdByName(const char *frame_name) { int id = SCMapEnumNameToValue(frame_name, tls_frame_table); @@ -3282,7 +3354,8 @@ void RegisterSSLParsers(void) AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TLS, STREAM_TOCLIENT, SSLParseServerRecord); - + AppLayerParserRegisterGetStateFuncs( + IPPROTO_TCP, ALPROTO_TLS, SSLStateGetStateIdByName, SSLStateGetStateNameById); AppLayerParserRegisterGetFrameFuncs( IPPROTO_TCP, ALPROTO_TLS, SSLStateGetFrameIdByName, SSLStateGetFrameNameById); AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_TLS, SSLStateGetEventInfo); From 66da9fb10a33039eedb4426fae52ae9de3e3d08e Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 23 Jan 2025 11:05:48 +0100 Subject: [PATCH 13/17] http1: register progress state names --- src/app-layer-htp.c | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index f76417a33fbe..460850570c2d 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -213,6 +213,82 @@ static const char *HTTPGetFrameNameById(const uint8_t frame_id) return name; } +static SCEnumCharMap http_state_client_table[] = { + { + "request_not_started", + HTP_REQUEST_NOT_STARTED, + }, + { + "request_line", + HTP_REQUEST_PROGRESS_LINE, + }, + { + "request_headers", + HTP_REQUEST_PROGRESS_HEADERS, + }, + { + "request_body", + HTP_REQUEST_PROGRESS_BODY, + }, + { + "request_trailer", + HTP_REQUEST_PROGRESS_TRAILER, + }, + { + "request_complete", + HTP_REQUEST_PROGRESS_COMPLETE, + }, + { NULL, -1 }, +}; + +static SCEnumCharMap http_state_server_table[] = { + { + "response_not_started", + HTP_RESPONSE_NOT_STARTED, + }, + { + "response_line", + HTP_RESPONSE_PROGRESS_LINE, + }, + { + "response_headers", + HTP_RESPONSE_PROGRESS_HEADERS, + }, + { + "response_body", + HTP_RESPONSE_PROGRESS_BODY, + }, + { + "response_trailer", + HTP_RESPONSE_PROGRESS_TRAILER, + }, + { + "response_complete", + HTP_RESPONSE_PROGRESS_COMPLETE, + }, + { NULL, -1 }, +}; + +static int HtpStateGetStateIdByName(const char *name, const uint8_t direction) +{ + SCEnumCharMap *map = + direction == STREAM_TOSERVER ? http_state_client_table : http_state_server_table; + + int id = SCMapEnumNameToValue(name, map); + if (id < 0) { + return -1; + } + return id; +} + +static const char *HtpStateGetStateNameById(const int id, const uint8_t direction) +{ + SCEnumCharMap *map = + direction == STREAM_TOSERVER ? http_state_client_table : http_state_server_table; + const char *name = SCMapEnumValueToName(id, map); + return name; +} + static void *HTPStateGetTx(void *alstate, uint64_t tx_id); static int HTPStateGetAlstateProgress(void *tx, uint8_t direction); static uint64_t HTPStateGetTxCnt(void *alstate); @@ -2873,6 +2949,9 @@ void RegisterHTPParsers(void) AppLayerParserRegisterGetFrameFuncs( IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById); /* app-layer-frame-documentation tag end: registering relevant callbacks */ + AppLayerParserRegisterGetStateFuncs( + IPPROTO_TCP, ALPROTO_HTTP1, HtpStateGetStateIdByName, HtpStateGetStateNameById); + HTPConfigure(); } else { SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name); From 11c1dc6ad6fd716ec131b02aaf485a28f0ee17f8 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 14 Jan 2025 09:41:11 +0100 Subject: [PATCH 14/17] WIP detect: introduce explicit hooks Generic: :request_done and :response_done Per protocol, it uses the registered progress (state) values. E.g. tls:client_hello_done A rule ruleset could be: pass tls:client_hello_done any any -> any any (tls.sni; content:"www.google.com"; sid:21; alert;) drop tls:client_hello_done any any -> any any (sid:22;) The pass rule is evaluated when the client hello is parsed, and if it doesn't match the drop rule will be evaluated. Registers each generic lists as "::generic" (e.g. "tls:client_hello_done:generic"). Ticket: #7485. --- src/detect-engine-build.c | 6 ++ src/detect-engine-mpm.c | 4 +- src/detect-engine-register.c | 2 + src/detect-engine.c | 36 ++++++- src/detect-parse.c | 190 ++++++++++++++++++++++++++++++++++- src/detect-parse.h | 2 + src/detect.c | 10 ++ src/detect.h | 23 +++++ 8 files changed, 267 insertions(+), 6 deletions(-) diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index 1793486ab1db..c89b3f2d997f 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -1644,6 +1644,12 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s) BUG_ON(s->type != SIG_TYPE_NOT_SET); int iponly = 0; + if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) { + s->type = SIG_TYPE_APP_TX; + SCLogNotice("%u: set to app_tx due to hook type app", s->id); + SCReturn; + } + /* see if the sig is dp only */ if (SignatureIsPDOnly(de_ctx, s) == 1) { s->type = SIG_TYPE_PDONLY; diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index fab8a661464b..3b5d8c2473d6 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -2514,7 +2514,7 @@ void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature * for (; app != NULL; app = app->next) { DEBUG_VALIDATE_BUG_ON(app->smd == NULL); SigMatchData *smd = app->smd; - do { + while (smd) { switch (smd->type) { case DETECT_CONTENT: { const DetectContentData *cd = (const DetectContentData *)smd->ctx; @@ -2542,7 +2542,7 @@ void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature * if (smd->is_last) break; smd++; - } while (1); + } } const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect; for (; pkt != NULL; pkt = pkt->next) { diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 658c17150aec..01c0be6ec8b5 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -503,6 +503,8 @@ void SigTableInit(void) void SigTableSetup(void) { + DetectRegisterAppLayerHookLists(); + DetectSidRegister(); DetectPriorityRegister(); DetectPrefilterRegister(); diff --git a/src/detect-engine.c b/src/detect-engine.c index 1979f709d671..1d323896172d 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -683,7 +683,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, new_engine->sm_list = t->sm_list; new_engine->sm_list_base = t->sm_list_base; new_engine->smd = smd; - new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd); + new_engine->match_on_null = smd ? DetectContentInspectionMatchOnAbsentBuffer(smd) : false; new_engine->progress = t->progress; new_engine->v2 = t->v2; SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback, @@ -749,6 +749,7 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature const int files_id = DetectBufferTypeGetByName("files"); bool head_is_mpm = false; uint8_t last_id = DE_STATE_FLAG_BASE; + SCLogNotice("%u: setup app inspect engines. %u buffers", s->id, s->init_data->buffer_index); for (uint32_t x = 0; x < s->init_data->buffer_index; x++) { SigMatchData *smd = SigMatchList2DataArray(s->init_data->buffers[x].head); @@ -788,6 +789,39 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature } } + /* handle rules that have an app-layer hook w/o bringing their own app inspect engine, + * e.g. `alert dns:request_complete ... (sid:1;)` + * + * Here we use a minimal stub inspect engine in which we set: + * - alproto + * - progress + * - sm_list/sm_list_base to get the mapping to the hook name + * - dir based on sig direction + * + * The inspect engine has no callback and is thus considered a straight match. + */ + if (s->init_data->buffer_index == 0 && s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) { + int dir = 0; + if ((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) == + (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) + abort(); + if ((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) == 0) + abort(); + if (s->flags & SIG_FLAG_TOSERVER) + dir = 0; + else if (s->flags & SIG_FLAG_TOCLIENT) + dir = 1; + + DetectEngineAppInspectionEngine t = { + .alproto = s->init_data->hook.t.app.alproto, + .progress = (uint16_t)s->init_data->hook.t.app.app_progress, + .sm_list = (uint16_t)s->init_data->hook.sm_list, + .sm_list_base = (uint16_t)s->init_data->hook.sm_list, + .dir = dir, + }; + AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm); + } + if ((s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) { diff --git a/src/detect-parse.c b/src/detect-parse.c index ab3f43b11e5d..4f86b452a96b 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1105,6 +1105,148 @@ static int SigParseAddress(DetectEngineCtx *de_ctx, return -1; } +static bool IsBuiltIn(const char *n) +{ + if (strcmp(n, "request_complete") == 0 || strcmp(n, "response_complete") == 0) { + return true; + } + return false; +} + +/** \brief register app hooks as generic lists + * + * Register each hook in each app protocol as: + * ::generic + * These lists can be used by lua scripts to hook into. + * + * \todo move elsewhere? maybe a detect-engine-hook.c? + */ +void DetectRegisterAppLayerHookLists(void) +{ + for (AppProto a = ALPROTO_FAILED + 1; a < g_alproto_max; a++) { + const char *alproto_name = AppProtoToString(a); + if (strcmp(alproto_name, "http") == 0) + alproto_name = "http1"; + SCLogDebug("alproto %u/%s", a, alproto_name); + + const int max_progress_ts = + AppLayerParserGetStateProgressCompletionStatus(a, STREAM_TOSERVER); + const int max_progress_tc = + AppLayerParserGetStateProgressCompletionStatus(a, STREAM_TOCLIENT); + + char ts_tx_complete[64]; + snprintf(ts_tx_complete, sizeof(ts_tx_complete), "%s:request_complete:generic", + alproto_name); + DetectAppLayerInspectEngineRegister(ts_tx_complete, a, SIG_FLAG_TOSERVER, max_progress_ts, + DetectEngineInspectGenericList, NULL); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "request_name", ts_tx_complete, + (uint32_t)strlen(ts_tx_complete)); + + char tc_tx_complete[64]; + snprintf(tc_tx_complete, sizeof(tc_tx_complete), "%s:response_complete:generic", + alproto_name); + DetectAppLayerInspectEngineRegister(tc_tx_complete, a, SIG_FLAG_TOCLIENT, max_progress_tc, + DetectEngineInspectGenericList, NULL); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "response_name", tc_tx_complete, + (uint32_t)strlen(tc_tx_complete)); + + for (int p = 0; p <= max_progress_ts; p++) { + const char *name = AppLayerParserGetStateNameById( + IPPROTO_TCP /* TODO no ipproto */, a, p, STREAM_TOSERVER); + if (name != NULL && !IsBuiltIn(name)) { + char list_name[64]; + snprintf(list_name, sizeof(list_name), "%s:%s:generic", alproto_name, name); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, name, list_name, + (uint32_t)strlen(list_name)); + + DetectAppLayerInspectEngineRegister( + list_name, a, SIG_FLAG_TOSERVER, p, DetectEngineInspectGenericList, NULL); + } + } + for (int p = 0; p <= max_progress_tc; p++) { + const char *name = AppLayerParserGetStateNameById( + IPPROTO_TCP /* TODO no ipproto */, a, p, STREAM_TOCLIENT); + if (name != NULL && !IsBuiltIn(name)) { + char list_name[64]; + snprintf(list_name, sizeof(list_name), "%s:%s:generic", alproto_name, name); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, name, list_name, + (uint32_t)strlen(list_name)); + + DetectAppLayerInspectEngineRegister( + list_name, a, SIG_FLAG_TOCLIENT, p, DetectEngineInspectGenericList, NULL); + } + } + } +} + +static const char *SignatureHookTypeToString(enum SignatureHookType t) +{ + switch (t) { + case SIGNATURE_HOOK_TYPE_NOT_SET: + return "not_set"; + case SIGNATURE_HOOK_TYPE_APP: + return "app"; + // case SIGNATURE_HOOK_TYPE_PKT: + // return "pkt"; + } + return "unknown"; +} + +static SignatureHook SetAppHook(const AppProto alproto, int progress) +{ + SignatureHook h = { + .type = SIGNATURE_HOOK_TYPE_APP, + .t.app.alproto = alproto, + .t.app.app_progress = progress, + }; + return h; +} + +/** + * \param proto_hook string of protocol and hook, e.g. dns:request_complete + */ +static int SigParseProtoHookApp(Signature *s, const char *proto_hook, const char *p, const char *h) +{ + if (strcmp(h, "request_complete") == 0) { + s->flags |= SIG_FLAG_TOSERVER; + s->init_data->hook = SetAppHook(s->alproto, + AppLayerParserGetStateProgressCompletionStatus(s->alproto, STREAM_TOSERVER)); + } else if (strcmp(h, "response_complete") == 0) { + s->flags |= SIG_FLAG_TOCLIENT; + s->init_data->hook = SetAppHook(s->alproto, + AppLayerParserGetStateProgressCompletionStatus(s->alproto, STREAM_TOCLIENT)); + } else { + const int progress_ts = AppLayerParserGetStateIdByName( + IPPROTO_TCP /* TODO */, s->alproto, h, STREAM_TOSERVER); + if (progress_ts >= 0) { + s->flags |= SIG_FLAG_TOSERVER; + s->init_data->hook = SetAppHook(s->alproto, progress_ts); + } else { + const int progress_tc = AppLayerParserGetStateIdByName( + IPPROTO_TCP /* TODO */, s->alproto, h, STREAM_TOCLIENT); + if (progress_tc < 0) { + return -1; + } + s->flags |= SIG_FLAG_TOCLIENT; + s->init_data->hook = SetAppHook(s->alproto, progress_tc); + } + } + + char generic_hook_name[64]; + snprintf(generic_hook_name, sizeof(generic_hook_name), "%s:generic", proto_hook); + int list = DetectBufferTypeGetByName(generic_hook_name); + if (list < 0) { + SCLogError("no list registered as %s for hook %s", generic_hook_name, proto_hook); + return -1; + } + s->init_data->hook.sm_list = list; + + SCLogNotice("protocol:%s hook:%s: type:%s alproto:%u hook:%d", p, h, + SignatureHookTypeToString(s->init_data->hook.type), s->init_data->hook.t.app.alproto, + s->init_data->hook.t.app.app_progress); + return 0; +} + /** * \brief Parses the protocol supplied by the Signature. * @@ -1120,15 +1262,41 @@ static int SigParseAddress(DetectEngineCtx *de_ctx, static int SigParseProto(Signature *s, const char *protostr) { SCEnter(); + if (strlen(protostr) > 32) + return -1; - int r = DetectProtoParse(&s->proto, (char *)protostr); + char proto[33]; + strlcpy(proto, protostr, 33); + const char *p = proto; + const char *h = NULL; + + bool has_hook = strchr(proto, ':') != NULL; + if (has_hook) { + char *xsaveptr = NULL; + p = strtok_r(proto, ":", &xsaveptr); + h = strtok_r(NULL, ":", &xsaveptr); + SCLogDebug("p: '%s' h: '%s'", p, h); + } + if (p == NULL) { + SCLogError("invalid protocol specification '%s'", proto); + return -1; + } + + int r = DetectProtoParse(&s->proto, p); if (r < 0) { - s->alproto = AppLayerGetProtoByName((char *)protostr); + s->alproto = AppLayerGetProtoByName(p); /* indicate that the signature is app-layer */ if (s->alproto != ALPROTO_UNKNOWN) { s->flags |= SIG_FLAG_APPLAYER; AppLayerProtoDetectSupportedIpprotos(s->alproto, s->proto.proto); + + if (h) { + if (SigParseProtoHookApp(s, protostr, p, h) < 0) { + SCLogError("protocol \"%s\" does not support hook \"%s\"", p, h); + SCReturnInt(-1); + } + } } else { SCLogError("protocol \"%s\" cannot be used " @@ -1136,7 +1304,7 @@ static int SigParseProto(Signature *s, const char *protostr) "is not yet supported OR detection has been disabled for " "protocol through the yaml option " "app-layer.protocols.%s.detection-enabled", - protostr, protostr); + p, p); SCReturnInt(-1); } } @@ -2058,6 +2226,22 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s) SCLogDebug("b->id %d nlists %d", b->id, nlists); bufdir[b->id].ts += (app->dir == 0); bufdir[b->id].tc += (app->dir == 1); + + /* only allow rules to use the hook for engines at that + * exact progress for now. */ + if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) { + if ((s->flags & SIG_FLAG_TOSERVER) && (app->dir == 0) && + app->progress != s->init_data->hook.t.app.app_progress) { + SCLogError("engine progress value %d doesn't match hook %u", app->progress, + s->init_data->hook.t.app.app_progress); + SCReturnInt(0); + } + if ((s->flags & SIG_FLAG_TOCLIENT) && (app->dir == 1) && + app->progress != s->init_data->hook.t.app.app_progress) { + SCLogError("engine progress value doesn't match hook"); + SCReturnInt(0); + } + } } } diff --git a/src/detect-parse.h b/src/detect-parse.h index ec2c204c0f42..6d00ec37cf59 100644 --- a/src/detect-parse.h +++ b/src/detect-parse.h @@ -120,4 +120,6 @@ int SC_Pcre2SubstringCopy( int SC_Pcre2SubstringGet(pcre2_match_data *match_data, uint32_t number, PCRE2_UCHAR **bufferptr, PCRE2_SIZE *bufflen); +void DetectRegisterAppLayerHookLists(void); + #endif /* SURICATA_DETECT_PARSE_H */ diff --git a/src/detect.c b/src/detect.c index e4be96958eb7..8c4c46c0cef5 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1048,6 +1048,16 @@ static bool DetectRunTxInspectRule(ThreadVars *tv, if (unlikely(engine->stream && can->stream_stored)) { match = can->stream_result; TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match); + } else if (engine->v2.Callback == NULL) { + /* TODO is this the cleanest way to support a non-app sig on a app hook? */ + + /* we don't have to store a "hook" match, also don't want to keep any state to make + * sure the hook gets invoked again until tx progress progresses. */ + if (tx->tx_progress <= engine->progress) + return DETECT_ENGINE_INSPECT_SIG_MATCH; + + /* if progress > engine progress, track state to avoid additional matches */ + match = DETECT_ENGINE_INSPECT_SIG_MATCH; } else { KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list); DEBUG_VALIDATE_BUG_ON(engine->v2.Callback == NULL); diff --git a/src/detect.h b/src/detect.h index f936e263ab6a..593bc94d1128 100644 --- a/src/detect.h +++ b/src/detect.h @@ -537,7 +537,30 @@ typedef struct SignatureInitDataBuffer_ { SigMatch *tail; } SignatureInitDataBuffer; +enum SignatureHookType { + SIGNATURE_HOOK_TYPE_NOT_SET, + // SIGNATURE_HOOK_TYPE_PKT, + SIGNATURE_HOOK_TYPE_APP, +}; + +// dns:request_complete should add DetectBufferTypeGetByName("dns:request_complete"); +// TODO to json +typedef struct SignatureHook_ { + enum SignatureHookType type; + int sm_list; /**< list id for the hook's generic list. e.g. for dns:request_complete:generic */ + union { + struct { + AppProto alproto; + /** progress value of the app-layer hook specified in the rule. Sets the app_proto + * specific progress value. */ + int app_progress; + } app; + } t; +} SignatureHook; + typedef struct SignatureInitData_ { + SignatureHook hook; + /** Number of sigmatches. Used for assigning SigMatch::idx */ uint16_t sm_cnt; From a2249fc1ce8f374c9d4f9a7721d5ee14d44a6a6d Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 19 Jan 2025 10:54:41 +0100 Subject: [PATCH 15/17] detect/lua: use rule hook instead of init() 'needs' table For registration of app-layer inspection, no longer use the 'needs' table from the script, but instead use the rule hook setting. Ticket: #4783. --- src/detect-lua.c | 348 ++++++++--------------------------------------- src/detect-lua.h | 2 - 2 files changed, 60 insertions(+), 290 deletions(-) diff --git a/src/detect-lua.c b/src/detect-lua.c index e5a8cb28c78d..435571ae1f03 100644 --- a/src/detect-lua.c +++ b/src/detect-lua.c @@ -102,24 +102,6 @@ void DetectLuaRegister(void) #define FLAG_DATATYPE_PACKET BIT_U32(0) #define FLAG_DATATYPE_PAYLOAD BIT_U32(1) #define FLAG_DATATYPE_STREAM BIT_U32(2) -#define FLAG_DATATYPE_HTTP_URI BIT_U32(3) -#define FLAG_DATATYPE_HTTP_URI_RAW BIT_U32(4) -#define FLAG_DATATYPE_HTTP_REQUEST_HEADERS BIT_U32(5) -#define FLAG_DATATYPE_HTTP_REQUEST_HEADERS_RAW BIT_U32(6) -#define FLAG_DATATYPE_HTTP_REQUEST_COOKIE BIT_U32(7) -#define FLAG_DATATYPE_HTTP_REQUEST_UA BIT_U32(8) -#define FLAG_DATATYPE_HTTP_REQUEST_LINE BIT_U32(9) -#define FLAG_DATATYPE_HTTP_REQUEST_BODY BIT_U32(10) -#define FLAG_DATATYPE_HTTP_RESPONSE_COOKIE BIT_U32(11) -#define FLAG_DATATYPE_HTTP_RESPONSE_BODY BIT_U32(12) -#define FLAG_DATATYPE_HTTP_RESPONSE_HEADERS BIT_U32(13) -#define FLAG_DATATYPE_HTTP_RESPONSE_HEADERS_RAW BIT_U32(14) -#define FLAG_DATATYPE_DNS_RRNAME BIT_U32(15) -#define FLAG_DATATYPE_DNS_REQUEST BIT_U32(16) -#define FLAG_DATATYPE_DNS_RESPONSE BIT_U32(17) -#define FLAG_DATATYPE_SSH BIT_U32(19) -#define FLAG_DATATYPE_SMTP BIT_U32(20) -#define FLAG_DATATYPE_DNP3 BIT_U32(21) #define FLAG_DATATYPE_BUFFER BIT_U32(22) #define FLAG_ERROR_LOGGED BIT_U32(23) #define FLAG_BLOCKED_FUNCTION_LOGGED BIT_U32(24) @@ -342,42 +324,10 @@ static int DetectLuaMatch (DetectEngineThreadCtx *det_ctx, SCReturnInt(0); if ((tlua->flags & FLAG_DATATYPE_PACKET) && GET_PKT_LEN(p) == 0) SCReturnInt(0); - if (tlua->alproto != ALPROTO_UNKNOWN) { - if (p->flow == NULL) - SCReturnInt(0); - - AppProto alproto = p->flow->alproto; - if (tlua->alproto != alproto) - SCReturnInt(0); - } lua_getglobal(tlua->luastate, "match"); lua_newtable(tlua->luastate); /* stack at -1 */ - if (tlua->alproto == ALPROTO_HTTP1) { - HtpState *htp_state = p->flow->alstate; - if (htp_state != NULL && htp_state->connp != NULL) { - htp_tx_t *tx = NULL; - uint64_t idx = AppLayerParserGetTransactionInspectId(p->flow->alparser, - STREAM_TOSERVER); - uint64_t total_txs= AppLayerParserGetTxCnt(p->flow, htp_state); - for ( ; idx < total_txs; idx++) { - tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, idx); - if (tx == NULL) - continue; - - if ((tlua->flags & FLAG_DATATYPE_HTTP_REQUEST_LINE) && - htp_tx_request_line(tx) != NULL && bstr_len(htp_tx_request_line(tx)) > 0) { - lua_pushliteral(tlua->luastate, "http.request_line"); /* stack at -2 */ - LuaPushStringBuffer(tlua->luastate, - (const uint8_t *)bstr_ptr(htp_tx_request_line(tx)), - bstr_len(htp_tx_request_line(tx))); - lua_settable(tlua->luastate, -3); - } - } - } - } - SCReturnInt(DetectLuaRunMatch(det_ctx, lua, tlua)); } @@ -397,33 +347,9 @@ static int DetectLuaAppMatchCommon (DetectEngineThreadCtx *det_ctx, /* setup extension data for use in lua c functions */ LuaExtensionsMatchSetup(tlua->luastate, lua, det_ctx, f, NULL, s, flags); - if (tlua->alproto != ALPROTO_UNKNOWN) { - int alproto = f->alproto; - if (tlua->alproto != alproto) - SCReturnInt(0); - } - lua_getglobal(tlua->luastate, "match"); lua_newtable(tlua->luastate); /* stack at -1 */ - if (tlua->alproto == ALPROTO_HTTP1) { - HtpState *htp_state = state; - if (htp_state != NULL && htp_state->connp != NULL) { - htp_tx_t *tx = NULL; - tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, det_ctx->tx_id); - if (tx != NULL) { - if ((tlua->flags & FLAG_DATATYPE_HTTP_REQUEST_LINE) && - htp_tx_request_line(tx) != NULL && bstr_len(htp_tx_request_line(tx)) > 0) { - lua_pushliteral(tlua->luastate, "http.request_line"); /* stack at -2 */ - LuaPushStringBuffer(tlua->luastate, - (const uint8_t *)bstr_ptr(htp_tx_request_line(tx)), - bstr_len(htp_tx_request_line(tx))); - lua_settable(tlua->luastate, -3); - } - } - } - } - SCReturnInt(DetectLuaRunMatch(det_ctx, lua, tlua)); } @@ -464,7 +390,6 @@ static void *DetectLuaThreadInit(void *data) return NULL; } - t->alproto = lua->alproto; t->flags = lua->flags; t->luastate = SCLuaSbStateNew(lua->alloc_limit, lua->instruction_limit); @@ -760,108 +685,12 @@ static int DetectLuaSetupPrime(DetectEngineCtx *de_ctx, DetectLuaData *ld, const SCLogError("alloc error"); goto error; } - - } else if (strncmp(k, "http", 4) == 0 && strcmp(v, "true") == 0) { - if (ld->alproto != ALPROTO_UNKNOWN && ld->alproto != ALPROTO_HTTP1) { - SCLogError( - "can just inspect script against one app layer proto like HTTP at a time"); - goto error; - } - if (ld->flags != 0) { - SCLogError("when inspecting HTTP buffers only a single buffer can be inspected"); - goto error; - } - - /* http types */ - ld->alproto = ALPROTO_HTTP1; - - if (strcmp(k, "http.uri") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_URI; - - else if (strcmp(k, "http.uri.raw") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_URI_RAW; - - else if (strcmp(k, "http.request_line") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_REQUEST_LINE; - - else if (strcmp(k, "http.request_headers") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_REQUEST_HEADERS; - - else if (strcmp(k, "http.request_headers.raw") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_REQUEST_HEADERS_RAW; - - else if (strcmp(k, "http.request_cookie") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_REQUEST_COOKIE; - - else if (strcmp(k, "http.request_user_agent") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_REQUEST_UA; - - else if (strcmp(k, "http.request_body") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_REQUEST_BODY; - - else if (strcmp(k, "http.response_body") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_RESPONSE_BODY; - - else if (strcmp(k, "http.response_cookie") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_RESPONSE_COOKIE; - - else if (strcmp(k, "http.response_headers") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_RESPONSE_HEADERS; - - else if (strcmp(k, "http.response_headers.raw") == 0) - ld->flags |= FLAG_DATATYPE_HTTP_RESPONSE_HEADERS_RAW; - - else { - SCLogError("unsupported http data type %s", k); - goto error; - } - - ld->buffername = SCStrdup(k); - if (ld->buffername == NULL) { - SCLogError("alloc error"); - goto error; - } - } else if (strncmp(k, "dns", 3) == 0 && strcmp(v, "true") == 0) { - - ld->alproto = ALPROTO_DNS; - - if (strcmp(k, "dns.rrname") == 0) - ld->flags |= FLAG_DATATYPE_DNS_RRNAME; - else if (strcmp(k, "dns.request") == 0) - ld->flags |= FLAG_DATATYPE_DNS_REQUEST; - else if (strcmp(k, "dns.response") == 0) - ld->flags |= FLAG_DATATYPE_DNS_RESPONSE; - - else { - SCLogError("unsupported dns data type %s", k); - goto error; - } - ld->buffername = SCStrdup(k); - if (ld->buffername == NULL) { - SCLogError("alloc error"); - goto error; - } - } else if (strncmp(k, "tls", 3) == 0 && strcmp(v, "true") == 0) { - - ld->alproto = ALPROTO_TLS; - - } else if (strncmp(k, "ssh", 3) == 0 && strcmp(v, "true") == 0) { - - ld->alproto = ALPROTO_SSH; - - ld->flags |= FLAG_DATATYPE_SSH; - - } else if (strncmp(k, "smtp", 4) == 0 && strcmp(v, "true") == 0) { - - ld->alproto = ALPROTO_SMTP; - - ld->flags |= FLAG_DATATYPE_SMTP; - - } else if (strncmp(k, "dnp3", 4) == 0 && strcmp(v, "true") == 0) { - - ld->alproto = ALPROTO_DNP3; - - ld->flags |= FLAG_DATATYPE_DNP3; + /* old options no longer supported */ + } else if (strncmp(k, "http", 4) == 0 || strncmp(k, "dns", 3) == 0 || + strncmp(k, "tls", 3) == 0 || strncmp(k, "ssh", 3) == 0 || + strncmp(k, "smtp", 4) == 0 || strncmp(k, "dnp3", 4) == 0) { + SCLogError("data type %s no longer supported, use rule hooks", k); + goto error; } else { SCLogError("unsupported data type %s", k); @@ -926,78 +755,22 @@ static int DetectLuaSetup (DetectEngineCtx *de_ctx, Signature *s, const char *st if (lua->thread_ctx_id == -1) goto error; - if (lua->alproto != ALPROTO_UNKNOWN) { - if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, lua->alproto)) { - goto error; + int list = DetectBufferGetActiveList(de_ctx, s); + SCLogNotice("buffer list %d -> %d", list, s->init_data->list); + if (list == -1 || (list == 0 && s->init_data->list == INT_MAX)) { + /* what needs to happen here is: we register to the rule hook, so e.g. + * http1.request_complete. This means we need a list. + * + * This includes each pkt, payload, stream, etc. */ + + if (s->init_data->hook.type != SIGNATURE_HOOK_TYPE_NOT_SET) { + list = s->init_data->hook.sm_list; + SCLogNotice("setting list %d", list); } - s->alproto = lua->alproto; - } - - /* Okay so far so good, lets get this into a SigMatch - * and put it in the Signature. */ - - int list = -1; - if (lua->alproto == ALPROTO_UNKNOWN) { - if (lua->flags & FLAG_DATATYPE_STREAM) - list = DETECT_SM_LIST_PMATCH; - else { - if (lua->flags & FLAG_DATATYPE_BUFFER) { - if (DetectBufferGetActiveList(de_ctx, s) != -1) { - list = s->init_data->list; - } else { - SCLogError("Lua and sticky buffer failure"); - goto error; - } - } else - list = DETECT_SM_LIST_MATCH; - } - - } else if (lua->alproto == ALPROTO_HTTP1) { - if (lua->flags & FLAG_DATATYPE_HTTP_RESPONSE_BODY) { - list = DetectBufferTypeGetByName("file_data"); - } else if (lua->flags & FLAG_DATATYPE_HTTP_REQUEST_BODY) { - list = DetectBufferTypeGetByName("http_client_body"); - } else if (lua->flags & FLAG_DATATYPE_HTTP_URI) { - list = DetectBufferTypeGetByName("http_uri"); - } else if (lua->flags & FLAG_DATATYPE_HTTP_URI_RAW) { - list = DetectBufferTypeGetByName("http_raw_uri"); - } else if (lua->flags & FLAG_DATATYPE_HTTP_REQUEST_COOKIE || - lua->flags & FLAG_DATATYPE_HTTP_RESPONSE_COOKIE) { - list = DetectBufferTypeGetByName("http_cookie"); - } else if (lua->flags & FLAG_DATATYPE_HTTP_REQUEST_UA) { - list = DetectBufferTypeGetByName("http_user_agent"); - } else if (lua->flags & - (FLAG_DATATYPE_HTTP_REQUEST_HEADERS | FLAG_DATATYPE_HTTP_RESPONSE_HEADERS)) { - list = DetectBufferTypeGetByName("http_header"); - } else if (lua->flags & (FLAG_DATATYPE_HTTP_REQUEST_HEADERS_RAW | - FLAG_DATATYPE_HTTP_RESPONSE_HEADERS_RAW)) { - list = DetectBufferTypeGetByName("http_raw_header"); - } else { - list = DetectBufferTypeGetByName("http_request_line"); - } - } else if (lua->alproto == ALPROTO_DNS) { - if (lua->flags & FLAG_DATATYPE_DNS_RRNAME) { - list = DetectBufferTypeGetByName("dns_query"); - } else if (lua->flags & FLAG_DATATYPE_DNS_REQUEST) { - list = DetectBufferTypeGetByName("dns_request"); - } else if (lua->flags & FLAG_DATATYPE_DNS_RESPONSE) { - list = DetectBufferTypeGetByName("dns_response"); - } - } else if (lua->alproto == ALPROTO_TLS) { - list = DetectBufferTypeGetByName("tls_generic"); - } else if (lua->alproto == ALPROTO_SSH) { - list = DetectBufferTypeGetByName("ssh_banner"); - } else if (lua->alproto == ALPROTO_SMTP) { - list = g_smtp_generic_list_id; - } else if (lua->alproto == ALPROTO_DNP3) { - list = DetectBufferTypeGetByName("dnp3"); - } else { - SCLogError("lua can't be used with protocol %s", AppLayerGetProtoName(lua->alproto)); - goto error; } if (list == -1) { - SCLogError("lua can't be used with protocol %s", AppLayerGetProtoName(lua->alproto)); + SCLogError("lua failed to set up"); // TODO how would we get here? goto error; } @@ -1070,35 +843,34 @@ static int LuaMatchTest01(void) { ConfSetFinal("security.lua.allow-rules", "true"); - const char script[] = - "function init (args)\n" - " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" - " needs[\"flowvar\"] = {\"cnt\"}\n" - " return needs\n" - "end\n" - "\n" - "function match(args)\n" - " a = ScFlowvarGet(0)\n" - " if a then\n" - " a = tostring(tonumber(a)+1)\n" - " print (a)\n" - " ScFlowvarSet(0, a, #a)\n" - " else\n" - " a = tostring(1)\n" - " print (a)\n" - " ScFlowvarSet(0, a, #a)\n" - " end\n" - " \n" - " print (\"pre check: \" .. (a))\n" - " if tonumber(a) == 2 then\n" - " print \"match\"\n" - " return 1\n" - " end\n" - " return 0\n" - "end\n" - "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + const char script[] = "function init (args)\n" + " local needs = {}\n" + " needs[\"flowvar\"] = {\"cnt\"}\n" + " return needs\n" + "end\n" + "\n" + "function match(args)\n" + " a = ScFlowvarGet(0)\n" + " if a then\n" + " a = tostring(tonumber(a)+1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " else\n" + " a = tostring(1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " end\n" + " \n" + " print (\"pre check: \" .. (a))\n" + " if tonumber(a) == 2 then\n" + " print \"match\"\n" + " return 1\n" + " end\n" + " return 0\n" + "end\n" + "return 0\n"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; @@ -1195,7 +967,6 @@ static int LuaMatchTest01a(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowvar\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -1220,7 +991,8 @@ static int LuaMatchTest01a(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; uint8_t httpbuf2[] = "POST / HTTP/1.1\r\n" @@ -1740,7 +1512,6 @@ static int LuaMatchTest04(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowint\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -1762,7 +1533,8 @@ static int LuaMatchTest04(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; uint8_t httpbuf2[] = "POST / HTTP/1.1\r\n" @@ -1856,7 +1628,6 @@ static int LuaMatchTest04a(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowint\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -1878,7 +1649,8 @@ static int LuaMatchTest04a(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; @@ -1974,7 +1746,6 @@ static int LuaMatchTest05(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowint\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -1989,7 +1760,8 @@ static int LuaMatchTest05(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; @@ -2085,7 +1857,6 @@ static int LuaMatchTest05a(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowint\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -2100,7 +1871,8 @@ static int LuaMatchTest05a(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; @@ -2196,7 +1968,6 @@ static int LuaMatchTest06(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowint\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -2216,7 +1987,8 @@ static int LuaMatchTest06(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; @@ -2312,7 +2084,6 @@ static int LuaMatchTest06a(void) { const char script[] = "function init (args)\n" " local needs = {}\n" - " needs[\"http.request_headers\"] = tostring(true)\n" " needs[\"flowint\"] = {\"cnt\"}\n" " return needs\n" "end\n" @@ -2332,7 +2103,8 @@ static int LuaMatchTest06a(void) " return 0\n" "end\n" "return 0\n"; - char sig[] = "alert http any any -> any any (flow:to_server; lua:unittest; sid:1;)"; + char sig[] = "alert http1:request_complete any any -> any any (flow:to_server; lua:unittest; " + "sid:1;)"; uint8_t httpbuf1[] = "POST / HTTP/1.1\r\n" "Host: www.emergingthreats.net\r\n\r\n"; diff --git a/src/detect-lua.h b/src/detect-lua.h index 6397c4c413df..b59cea784460 100644 --- a/src/detect-lua.h +++ b/src/detect-lua.h @@ -30,7 +30,6 @@ typedef struct DetectLuaThreadData { lua_State *luastate; uint32_t flags; - int alproto; } DetectLuaThreadData; #define DETECT_LUA_MAX_FLOWVARS 15 @@ -42,7 +41,6 @@ typedef struct DetectLuaData { int negated; char *filename; uint32_t flags; - AppProto alproto; char *buffername; /* buffer name in case of a single buffer */ uint32_t flowint[DETECT_LUA_MAX_FLOWINTS]; uint16_t flowints; From f4874cd4cf45f7df1992bd8b0fe0477eebae6671 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 20 Jan 2025 13:05:32 +0100 Subject: [PATCH 16/17] detect: reuse hook based generic lists --- src/detect-dns-query.c | 10 ---------- src/detect-tls-cert-validity.c | 2 +- src/detect-tls-certs.c | 5 +---- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/detect-dns-query.c b/src/detect-dns-query.c index f50267566480..e28f890a0eec 100644 --- a/src/detect-dns-query.c +++ b/src/detect-dns-query.c @@ -115,16 +115,6 @@ void DetectDnsQueryRegister (void) DetectBufferTypeSupportsMultiInstance("dns_query"); g_dns_query_buffer_id = DetectBufferTypeGetByName("dns_query"); - - /* register these generic engines from here for now */ - DetectAppLayerInspectEngineRegister( - "dns_request", ALPROTO_DNS, SIG_FLAG_TOSERVER, 1, DetectEngineInspectGenericList, NULL); - DetectAppLayerInspectEngineRegister("dns_response", ALPROTO_DNS, SIG_FLAG_TOCLIENT, 1, - DetectEngineInspectGenericList, NULL); - - DetectBufferTypeSetDescriptionByName("dns_request", - "dns requests"); - DetectBufferTypeSetDescriptionByName("dns_response", "dns responses"); } diff --git a/src/detect-tls-cert-validity.c b/src/detect-tls-cert-validity.c index 9e858dc85112..0af00a56a676 100644 --- a/src/detect-tls-cert-validity.c +++ b/src/detect-tls-cert-validity.c @@ -123,7 +123,7 @@ void DetectTlsValidityRegister (void) DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); - g_tls_validity_buffer_id = DetectBufferTypeGetByName("tls_validity"); + g_tls_validity_buffer_id = DetectBufferTypeGetByName("tls:server_cert_done:generic"); } /** diff --git a/src/detect-tls-certs.c b/src/detect-tls-certs.c index 1f2ef564fc69..9b5d81561d06 100644 --- a/src/detect-tls-certs.c +++ b/src/detect-tls-certs.c @@ -157,7 +157,7 @@ static int DetectTlsCertsSetup(DetectEngineCtx *de_ctx, Signature *s, } static int g_tls_cert_buffer_id = 0; -#define BUFFER_NAME "tls_validity" +#define BUFFER_NAME "tls:server_cert_done:generic" #define KEYWORD_ID DETECT_TLS_CHAIN_LEN #define KEYWORD_NAME "tls.cert_chain_len" #define KEYWORD_DESC "match TLS certificate chain length" @@ -252,9 +252,6 @@ void DetectTlsCertChainLenRegister(void) sigmatch_table[KEYWORD_ID].Setup = DetectTLSCertChainLenSetup; sigmatch_table[KEYWORD_ID].Free = DetectTLSCertChainLenFree; - DetectAppLayerInspectEngineRegister(BUFFER_NAME, ALPROTO_TLS, SIG_FLAG_TOCLIENT, - TLS_STATE_SERVER_CERT_DONE, DetectEngineInspectGenericList, NULL); - g_tls_cert_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME); } From ef1291b86f36ff54da92503c97a70116aee0d7be Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 23 Jan 2025 16:11:30 +0100 Subject: [PATCH 17/17] WIP start of pkt hook --- src/detect-parse.c | 68 ++++++++++++++++++++++++++++++++++++++++++++-- src/detect.h | 10 ++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/detect-parse.c b/src/detect-parse.c index 4f86b452a96b..7673a80e62e3 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1186,12 +1186,61 @@ static const char *SignatureHookTypeToString(enum SignatureHookType t) return "not_set"; case SIGNATURE_HOOK_TYPE_APP: return "app"; - // case SIGNATURE_HOOK_TYPE_PKT: - // return "pkt"; + case SIGNATURE_HOOK_TYPE_PKT: + return "pkt"; } return "unknown"; } +static enum SignatureHookPkt HookPktFromString(const char *str) +{ + if (strcmp(str, "flow_start") == 0) { + return SIGNATURE_HOOK_PKT_FLOW_START; + } + return SIGNATURE_HOOK_PKT_NOT_SET; +} + +static const char *HookPktToString(const enum SignatureHookPkt ph) +{ + switch (ph) { + case SIGNATURE_HOOK_PKT_NOT_SET: + return "not set"; + case SIGNATURE_HOOK_PKT_FLOW_START: + return "flow_start"; + } + return "error"; +} + +static SignatureHook SetPktHook(const char *hook_str) +{ + SignatureHook h = { + .type = SIGNATURE_HOOK_TYPE_PKT, + .t.pkt.ph = HookPktFromString(hook_str), + }; + return h; +} + +/** + * \param proto_hook string of protocol and hook, e.g. dns:request_complete + */ +static int SigParseProtoHookPkt(Signature *s, const char *proto_hook, const char *p, const char *h) +{ + if (strcmp(h, "flow_start") == 0) { + s->init_data->hook = SetPktHook(h); + if (s->init_data->hook.t.pkt.ph == SIGNATURE_HOOK_PKT_NOT_SET) { + return -1; + } + } else { + SCLogError("unknown pkt hook %s", h); + } + // s->init_data->hook.sm_list = list; + + SCLogNotice("protocol:%s hook:%s: type:%s parsed hook:%s", p, h, + SignatureHookTypeToString(s->init_data->hook.type), + HookPktToString(s->init_data->hook.t.pkt.ph)); + return 0; +} + static SignatureHook SetAppHook(const AppProto alproto, int progress) { SignatureHook h = { @@ -1307,6 +1356,13 @@ static int SigParseProto(Signature *s, const char *protostr) p, p); SCReturnInt(-1); } + } else if (h != NULL) { + SCLogNotice("non-app-layer rule with %s:%s", p, h); + + if (SigParseProtoHookPkt(s, protostr, p, h) < 0) { + SCLogError("protocol \"%s\" does not support hook \"%s\"", p, h); + SCReturnInt(-1); + } } /* if any of these flags are set they are set in a mutually exclusive @@ -2477,6 +2533,14 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr, } } + if (sig->init_data->hook.type == SIGNATURE_HOOK_TYPE_PKT) { + if (sig->init_data->hook.t.pkt.ph == SIGNATURE_HOOK_PKT_FLOW_START) { + if ((sig->flags & SIG_FLAG_TOSERVER) != 0) { + sig->init_data->init_flags |= SIG_FLAG_INIT_FLOW; + } + } + } + if (!(sig->init_data->init_flags & SIG_FLAG_INIT_FLOW)) { if ((sig->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) { sig->flags |= SIG_FLAG_TOSERVER; diff --git a/src/detect.h b/src/detect.h index 593bc94d1128..6b8793b44bcb 100644 --- a/src/detect.h +++ b/src/detect.h @@ -537,9 +537,14 @@ typedef struct SignatureInitDataBuffer_ { SigMatch *tail; } SignatureInitDataBuffer; +enum SignatureHookPkt { + SIGNATURE_HOOK_PKT_NOT_SET, + SIGNATURE_HOOK_PKT_FLOW_START, +}; + enum SignatureHookType { SIGNATURE_HOOK_TYPE_NOT_SET, - // SIGNATURE_HOOK_TYPE_PKT, + SIGNATURE_HOOK_TYPE_PKT, SIGNATURE_HOOK_TYPE_APP, }; @@ -555,6 +560,9 @@ typedef struct SignatureHook_ { * specific progress value. */ int app_progress; } app; + struct { + enum SignatureHookPkt ph; + } pkt; } t; } SignatureHook;