From 5e2cb4cc497afaa36079ad32aa2f5c651ab515e3 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Thu, 28 Dec 2023 22:11:56 +0100
Subject: [PATCH] detect: move non-pf rules into special prefilter engines

Instead of having a per detection engine list of rule that couldn't be
prefiltered, put those into special "prefilter" engines.

For packet and frame rules this doesn't change much, it just removes
some hard coded logic from the detect engine.

For the packet non-prefilter rules in the "non-prefilter" special prefilter
engine, add additional filtering for the packet variant. It can prefilter on
alproto, dsize and dest port.

For app-layer, there is an engine per progress value, per app-layer
protocol and per direction. This hooks app-layer non-prefilter rules
into the app inspect logic at the correct "progress" hook.

e.g. a rule like
        dns.query; bsize:1;

Negated MPM rules will also fall into this category:
        dns.query; content:!"abc";

Are part of a special "generic list" app engine for dns, at the
same progress hook as `dns.query`.

This all results in a lot fewer checks:

previous:

  --------------------------------------------------------------------------
  Date: 1/29/2025 -- 10:22:25. Sorted by: number of checks.
  --------------------------------------------------------------------------
   Num      Rule         Gid      Rev      Ticks        %      Checks   Matches  Max Ticks   Avg Ticks   Avg Match   Avg No Match
  -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- --------------
  1        20           1        0        181919672    11.85  588808   221      60454       308.96      2691.46     308.07
  2        50           1        0        223455914    14.56  453104   418      61634       493.17      3902.59     490.02
  3        60           1        0        185990683    12.12  453104   418      60950       410.48      1795.40     409.20
  4        51           1        0        192436011    12.54  427028   6084     61223       450.64      2749.12     417.42
  5        61           1        0        180401533    11.75  427028   6084     61093       422.46      2177.04     397.10
  6        70           1        0        153899099    10.03  369836   0        61282       416.13      0.00        416.13
  7        71           1        0        123389405    8.04   369836   12833    44921       333.63      2430.23     258.27
  8        41           1        0        63889876     4.16   155824   12568    39138       410.01      1981.97     272.10
  9        40           1        0        64149724     4.18   155818   210      39792       411.70      4349.57     406.38
  10       10           1        0        70848850     4.62   65558    0        39544       1080.70     0.00        1080.70
  11       11           1        0        94743878     6.17   65558    32214    60547       1445.19     2616.14     313.92

this commit:

  --------------------------------------------------------------------------
  Date: 1/29/2025 -- 10:15:46. Sorted by: number of checks.
  --------------------------------------------------------------------------
   Num      Rule         Gid      Rev      Ticks        %      Checks   Matches  Max Ticks   Avg Ticks   Avg Match   Avg No Match
  -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- --------------
  1        50           1        0        138776766    19.23  95920    418      167584      1446.80     3953.11     1435.83
  2        60           1        0        97988084     13.58  95920    418      182817      1021.56     1953.63     1017.48
  3        51           1        0        105318318    14.60  69838    6084     65649       1508.04     2873.38     1377.74
  4        61           1        0        89571260     12.41  69838    6084     164632      1282.56     2208.41     1194.20
  5        11           1        0        91132809     12.63  32779    32214    373569      2780.22     2785.58     2474.45
  6        10           1        0        66095303     9.16   32779    0        56704       2016.39     0.00        2016.39
  7        70           1        0        48107573     6.67   12928    0        42832       3721.19     0.00        3721.19
  8        71           1        0        32308792     4.48   12928    12833    39565       2499.13     2510.05     1025.09
  9        41           1        0        25546837     3.54   12886    12470    41479       1982.53     1980.84     2033.05
  10       40           1        0        26069992     3.61   12886    210      38495       2023.13     4330.05     1984.91
  11       20           1        0        639025       0.09   221      221      14750       2891.52     2891.52     0.00
---
 src/app-layer-frames.h          |   2 +
 src/decode.h                    |   4 +
 src/detect-engine-build.c       |   4 +-
 src/detect-engine-frame.c       |   5 +-
 src/detect-engine-prefilter.c   | 428 +++++++++++++++++++++++++++++++-
 src/detect-engine-prefilter.h   |   3 +-
 src/detect-engine-profile.c     |   6 +-
 src/detect-engine-siggroup.c    |  88 -------
 src/detect-engine.c             |  21 +-
 src/detect.c                    | 196 ++-------------
 src/detect.h                    |  24 +-
 src/packet.c                    |   1 +
 src/util-profiling-rulegroups.c |   3 +-
 13 files changed, 469 insertions(+), 316 deletions(-)

diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
index f1d5135d8953..e2eb2d461236 100644
--- a/src/app-layer-frames.h
+++ b/src/app-layer-frames.h
@@ -26,6 +26,8 @@
 
 #include "rust.h"
 
+/** special value for matching any type */
+#define FRAME_ANY_TYPE 62
 /** max 63 to fit the 64 bit per protocol space */
 #define FRAME_STREAM_TYPE 63
 
diff --git a/src/decode.h b/src/decode.h
index 1b299864a7c0..d7344da82a9a 100644
--- a/src/decode.h
+++ b/src/decode.h
@@ -94,6 +94,9 @@ enum PktSrcEnum {
 
 #include "util-validate.h"
 
+/* for now a uint8_t is enough -- here in decode as it's part of the packet */
+#define SignatureMask uint8_t
+
 /* forward declarations */
 struct DetectionEngineThreadCtx_;
 typedef struct AppLayerThreadCtx_ AppLayerThreadCtx;
@@ -508,6 +511,7 @@ typedef struct Packet_
     /* coccinelle: Packet:flowflags:FLOW_PKT_ */
 
     uint8_t app_update_direction; // enum StreamUpdateDir
+    SignatureMask sig_mask;
 
     /* Pkt Flags */
     uint32_t flags;
diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index af2ad13c38eb..77696001b9ea 100644
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -1990,8 +1990,6 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
 
         PrefilterSetupRuleGroup(de_ctx, sgh);
 
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
-
         sgh->id = idx;
         cnt++;
     }
@@ -2002,7 +2000,7 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
     if (de_ctx->decoder_event_sgh != NULL) {
         /* no need to set filestore count here as that would make a
          * signature not decode event only. */
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, de_ctx->decoder_event_sgh);
+        PrefilterSetupRuleGroup(de_ctx, de_ctx->decoder_event_sgh);
     }
 
     int dump_grouping = 0;
diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c
index fd3163d59732..85c64772aac1 100644
--- a/src/detect-engine-frame.c
+++ b/src/detect-engine-frame.c
@@ -76,8 +76,9 @@ void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead
     SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
     PrefilterEngine *engine = sgh->frame_engines;
     do {
-        BUG_ON(engine->alproto == ALPROTO_UNKNOWN);
-        if (engine->alproto == alproto && engine->ctx.frame_type == frame->type) {
+        if ((engine->alproto == alproto || engine->alproto == ALPROTO_UNKNOWN) &&
+                (engine->ctx.frame_type == frame->type ||
+                        engine->ctx.frame_type == FRAME_ANY_TYPE)) {
             SCLogDebug("frame %p engine %p", frame, engine);
             PREFILTER_PROFILING_START(det_ctx);
             engine->cb.PrefilterFrame(det_ctx, engine->pectx, p, frames, frame);
diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c
index db388bd6d70d..51531c2667f1 100644
--- a/src/detect-engine-prefilter.c
+++ b/src/detect-engine-prefilter.c
@@ -51,12 +51,14 @@
 #include "detect-engine-prefilter.h"
 #include "detect-engine-mpm.h"
 #include "detect-engine-frame.h"
+#include "detect-engine-uint.h"
 
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
 
 #include "util-profiling.h"
 #include "util-validate.h"
+#include "util-hash-string.h"
 
 static int PrefilterStoreGetId(DetectEngineCtx *de_ctx,
         const char *name, void (*FreeFunc)(void *));
@@ -127,6 +129,9 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
         PREFILTER_PROFILING_END(det_ctx, engine->gid);
 
         if (tx->tx_progress > engine->ctx.tx_min_progress && engine->is_last_for_progress) {
+            SCLogDebug("tx->tx_progress %d engine->ctx.tx_min_progress %d "
+                       "engine->is_last_for_progress %d",
+                    tx->tx_progress, engine->ctx.tx_min_progress, engine->is_last_for_progress);
             tx->prefilter_flags |= BIT_U64(engine->ctx.tx_min_progress);
         }
     next:
@@ -413,7 +418,181 @@ static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b)
     }
 }
 
-void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+struct PrefilterNonPFDataSig {
+    uint32_t sid : 30;
+    uint32_t type : 2; // 0 alproto 1 dport 2 dsize
+    uint16_t value;
+    union {
+        struct {
+            SignatureMask sig_mask;
+        } pkt;
+        struct {
+            uint8_t type; // TODO
+        } frame;
+        struct {
+            uint8_t foo; // TODO
+        } app;
+    };
+};
+
+struct PrefilterNonPFData {
+    uint32_t size;
+    struct PrefilterNonPFDataSig array[];
+};
+
+static void PrefilterTxNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f,
+        void *tx, const uint64_t tx_id, const AppLayerTxData *tx_data, const uint8_t flags)
+{
+    const uint16_t alproto = f->alproto;
+    const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx;
+    SCLogDebug("adding %u sids", data->size);
+    for (uint32_t i = 0; i < data->size; i++) {
+        const struct PrefilterNonPFDataSig *ds = &data->array[i];
+        if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) {
+            const uint32_t sid = ds->sid;
+            PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+        }
+    }
+}
+
+#ifdef NONPF_PKT_STATS
+static thread_local uint64_t prefilter_pkt_nonpf_called = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_mask_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_alproto_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_dsize_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_dport_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_sids = 0;
+#define NONPF_PKT_STATS_INCR(s) (s)++
+#else
+#define NONPF_PKT_STATS_INCR(s)
+#endif
+
+void PrefilterPktNonPFStatsDump(void)
+{
+#ifdef NONPF_PKT_STATS
+    SCLogNotice("prefilter non-pf: called:%" PRIu64 ", mask_fail:%" PRIu64 ", alproto fail:%" PRIu64
+                ", dport fail:%" PRIu64 ", dsize fail:%" PRIu64 ", sids:%" PRIu64
+                ", avg sids:%" PRIu64,
+            prefilter_pkt_nonpf_called, prefilter_pkt_nonpf_mask_fail,
+            prefilter_pkt_nonpf_alproto_fail, prefilter_pkt_nonpf_dport_fail,
+            prefilter_pkt_nonpf_dsize_fail, prefilter_pkt_nonpf_sids,
+            prefilter_pkt_nonpf_called ? prefilter_pkt_nonpf_sids / prefilter_pkt_nonpf_called : 0);
+#endif
+}
+
+/* TODO the old code checked alproto and mask */
+static void PrefilterPktNonPF(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
+{
+    const uint16_t alproto = p->flow ? p->flow->alproto : ALPROTO_UNKNOWN;
+    const SignatureMask mask = p->sig_mask;
+    const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx;
+    SCLogDebug("adding %u sids", data->size);
+    NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_called);
+    for (uint32_t i = 0; i < data->size; i++) {
+        const struct PrefilterNonPFDataSig *ds = &data->array[i];
+        const SignatureMask rule_mask = ds->pkt.sig_mask;
+        if ((rule_mask & mask) == rule_mask) {
+            switch (ds->type) {
+                case 0:
+                    if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) {
+                        const uint32_t sid = ds->sid;
+                        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids);
+                    } else {
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_alproto_fail);
+                    }
+                    break;
+                case 1:
+                    if (ds->value == p->dp) {
+                        const uint32_t sid = ds->sid;
+                        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids);
+                    } else {
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_dport_fail);
+                    }
+                    break;
+                case 2:
+                    if (ds->value == p->payload_len) {
+                        const uint32_t sid = ds->sid;
+                        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids);
+                    } else {
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_dsize_fail);
+                    }
+                    break;
+            }
+        } else {
+            NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_mask_fail);
+        }
+    }
+}
+
+static void PrefilterFrameNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p,
+        const Frames *frames, const Frame *frame)
+{
+    DEBUG_VALIDATE_BUG_ON(p->flow == NULL);
+    const uint16_t alproto = p->flow->alproto;
+    const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx;
+    SCLogDebug("adding %u sids", data->size);
+    for (uint32_t i = 0; i < data->size; i++) {
+        const struct PrefilterNonPFDataSig *ds = &data->array[i];
+        if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) {
+            const uint32_t sid = ds->sid;
+            PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+        }
+    }
+}
+
+static uint32_t NonPFNamesHash(HashTable *h, void *data, uint16_t _len)
+{
+    const char *str = data;
+    return StringHashDjb2((const uint8_t *)str, (uint16_t)strlen(str)) % h->array_size;
+}
+
+static char NonPFNamesCompare(void *data1, uint16_t _len1, void *data2, uint16_t len2)
+{
+    const char *s1 = data1;
+    const char *s2 = data2;
+    return StringHashCompareFunc(data1, (uint16_t)strlen(s1), data2, (uint16_t)strlen(s2));
+}
+
+static void NonPFNamesFree(void *data)
+{
+    SCFree(data);
+}
+
+struct TxNonPFData {
+    AppProto alproto;
+    int dir; // 0 toserver 1 toclient
+    int sm_list;
+    int progress;
+    uint32_t sigs_cnt;
+    struct PrefilterNonPFDataSig *sigs;
+    const char *app_name;
+};
+
+static uint32_t TxNonPFHash(HashListTable *h, void *data, uint16_t _len)
+{
+    struct TxNonPFData *d = data;
+    return (d->alproto + d->sm_list + d->progress + d->dir) % h->array_size;
+}
+
+static char TxNonPFCompare(void *data1, uint16_t _len1, void *data2, uint16_t len2)
+{
+    struct TxNonPFData *d1 = data1;
+    struct TxNonPFData *d2 = data2;
+    return d1->alproto == d2->alproto && d1->sm_list == d2->sm_list &&
+           d1->progress == d2->progress && d1->dir == d2->dir;
+}
+
+static void TxNonPFFree(void *data)
+{
+    struct TxNonPFData *d = data;
+    SCFree(d->sigs);
+    SCFree(d);
+}
+
+int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 {
     int r = PatternMatchPrepareGroup(de_ctx, sgh);
     if (r != 0) {
@@ -434,6 +613,236 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
     }
 
+    const uint32_t max_sids = DetectEngineGetMaxSigId(de_ctx);
+    SCLogDebug("max_sids %u", max_sids);
+    struct PrefilterNonPFDataSig *pkt_non_pf_array = SCCalloc(max_sids, sizeof(*pkt_non_pf_array));
+    uint32_t pkt_non_pf_array_size = 0;
+    struct PrefilterNonPFDataSig *frame_non_pf_array =
+            SCCalloc(max_sids, sizeof(*frame_non_pf_array));
+    uint32_t frame_non_pf_array_size = 0;
+
+    HashListTable *tx_engines_hash =
+            HashListTableInit(256, TxNonPFHash, TxNonPFCompare, TxNonPFFree);
+    if (tx_engines_hash == NULL) {
+        SCFree(pkt_non_pf_array);
+        SCFree(frame_non_pf_array);
+        return -1;
+    }
+
+    if (de_ctx->non_pf_engine_names == NULL) {
+        de_ctx->non_pf_engine_names =
+                HashTableInit(512, NonPFNamesHash, NonPFNamesCompare, NonPFNamesFree);
+        BUG_ON(de_ctx->non_pf_engine_names == NULL);
+    }
+
+#ifdef NONPF_PKT_STATS
+    uint32_t nonpf_pkt_alproto = 0;
+    uint32_t nonpf_pkt_dsize = 0;
+    uint32_t nonpf_pkt_dport = 0;
+#endif
+    for (uint32_t sig = 0; sig < sgh->init->sig_cnt; sig++) {
+        Signature *s = sgh->init->match_array[sig];
+        if (s == NULL)
+            continue;
+        if (s->init_data->mpm_sm != NULL && (s->flags & SIG_FLAG_MPM_NEG) == 0)
+            continue;
+        if (s->init_data->prefilter_sm != NULL)
+            continue;
+        if ((s->flags & (SIG_FLAG_PREFILTER | SIG_FLAG_MPM_NEG)) == SIG_FLAG_PREFILTER)
+            continue;
+        SCLogDebug("setting up sid %u for non-prefilter", s->id);
+
+        bool tx_non_pf = false;
+        bool frame_non_pf = false;
+        bool pkt_non_pf = false;
+        for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
+            const int list_id = s->init_data->buffers[x].id;
+            const DetectBufferType *buf = DetectEngineBufferTypeGetById(de_ctx, list_id);
+            if (buf->frame) {
+                frame_non_pf = true;
+            } else if (buf->packet) {
+                pkt_non_pf = true;
+            } else {
+                SCLogDebug("x %u list_id %d", x, list_id);
+                for (DetectEngineAppInspectionEngine *app = de_ctx->app_inspect_engines;
+                        app != NULL; app = app->next) {
+                    if (!((((s->flags & SIG_FLAG_TOSERVER) != 0 && app->dir == 0) ||
+                                  ((s->flags & SIG_FLAG_TOCLIENT) != 0 && app->dir == 1)) &&
+                                list_id == (int)app->sm_list &&
+                                AppProtoEquals(s->alproto, app->alproto)))
+                        continue;
+
+                    struct TxNonPFData lookup = {
+                        .alproto = app->alproto,
+                        .dir = app->dir,
+                        .sm_list = app->sm_list,
+                        .progress = app->progress,
+                        .sigs_cnt = 0,
+                        .sigs = NULL,
+                        .app_name = NULL,
+                    };
+                    struct TxNonPFData *e = HashListTableLookup(tx_engines_hash, &lookup, 0);
+                    if (e != NULL) {
+                        BUG_ON(e->sigs_cnt == max_sids);
+                        e->sigs[e->sigs_cnt].sid = s->num;
+                        e->sigs[e->sigs_cnt].value = app->alproto;
+                        e->sigs_cnt++;
+                    } else {
+                        struct TxNonPFData *add = SCCalloc(1, sizeof(*add));
+                        if (add == NULL) {
+                            goto error;
+                        }
+                        add->dir = app->dir;
+                        add->alproto = app->alproto;
+                        add->sm_list = app->sm_list;
+                        add->progress = app->progress;
+                        add->sigs = SCCalloc(max_sids, sizeof(struct PrefilterNonPFDataSig));
+                        if (add->sigs == NULL) {
+                            SCFree(add);
+                            goto error;
+                        }
+                        add->sigs_cnt = 0;
+                        add->sigs[add->sigs_cnt].sid = s->num;
+                        add->sigs[add->sigs_cnt].value = app->alproto;
+                        add->sigs_cnt++;
+
+                        char engine_name[80];
+                        snprintf(engine_name, sizeof(engine_name), "%s:%s:non_pf",
+                                AppProtoToString(app->alproto), buf->name);
+                        char *engine_name_heap = SCStrdup(engine_name);
+                        BUG_ON(engine_name_heap == NULL);
+                        int result = HashTableAdd(de_ctx->non_pf_engine_names, engine_name_heap,
+                                (uint16_t)strlen(engine_name_heap));
+                        BUG_ON(result != 0);
+
+                        add->app_name = engine_name_heap;
+                        SCLogDebug("engine_name_heap %s", engine_name_heap);
+
+                        int ret = HashListTableAdd(tx_engines_hash, add, 0);
+                        if (ret != 0) {
+                            SCFree(add->sigs);
+                            SCFree(add);
+                            goto error;
+                        }
+                    }
+                    tx_non_pf = true;
+                }
+            }
+        }
+
+        if (!(tx_non_pf || frame_non_pf)) {
+            pkt_non_pf = true;
+        }
+
+        SCLogDebug("setting up sid %u for non-prefilter: %s", s->id,
+                tx_non_pf ? "tx engine" : (frame_non_pf ? "frame engine" : "pkt engine"));
+        if (tx_non_pf) {
+            // done above
+        } else if (pkt_non_pf) {
+            /* for pkt non prefilter, we have some space in the structure,
+             * so we can squeeze another filter */
+            uint8_t type;
+            uint16_t value;
+            if ((s->flags & SIG_FLAG_DSIZE) && s->dsize_mode == DETECT_UINT_EQ) {
+                type = 2;
+                value = s->dsize_low;
+#ifdef NONPF_PKT_STATS
+                nonpf_pkt_dsize++;
+#endif
+            } else if (s->dp != NULL && s->dp->next == NULL && s->dp->port == s->dp->port2) {
+                type = 1;
+                value = s->dp->port;
+#ifdef NONPF_PKT_STATS
+                nonpf_pkt_dport++;
+#endif
+            } else {
+                type = 0;
+                value = s->alproto;
+#ifdef NONPF_PKT_STATS
+                nonpf_pkt_alproto++;
+#endif
+            }
+
+            pkt_non_pf_array[pkt_non_pf_array_size].sid = s->num;
+            pkt_non_pf_array[pkt_non_pf_array_size].value = value;
+            pkt_non_pf_array[pkt_non_pf_array_size].type = type;
+            pkt_non_pf_array[pkt_non_pf_array_size].pkt.sig_mask = s->mask;
+            pkt_non_pf_array_size++;
+        } else if (frame_non_pf) {
+            frame_non_pf_array[frame_non_pf_array_size].sid = s->num;
+            frame_non_pf_array[frame_non_pf_array_size].value = s->alproto;
+            frame_non_pf_array_size++;
+        } else {
+            BUG_ON(1);
+        }
+    }
+
+    /* for each unique sig set, add an engine */
+    for (HashListTableBucket *b = HashListTableGetListHead(tx_engines_hash); b != NULL;
+            b = HashListTableGetListNext(b)) {
+        struct TxNonPFData *t = HashListTableGetListData(b);
+        SCLogDebug("%s engine for %s list %d hook %d has %u non-pf sigs",
+                t->dir == 0 ? "toserver" : "toclient", AppProtoToString(t->alproto), t->sm_list,
+                t->progress, t->sigs_cnt);
+
+        struct PrefilterNonPFData *data =
+                SCCalloc(1, sizeof(*data) + t->sigs_cnt * sizeof(data->array[0]));
+        if (data == NULL)
+            goto error;
+        data->size = t->sigs_cnt;
+        memcpy((uint8_t *)&data->array, t->sigs, t->sigs_cnt * sizeof(data->array[0]));
+        // TODO proper free func
+        if (PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxNonPF, t->alproto, t->progress,
+                    (void *)data, free, t->app_name) < 0) {
+            SCFree(data);
+            goto error;
+        }
+    }
+    HashListTableFree(tx_engines_hash);
+    tx_engines_hash = NULL;
+
+    if (pkt_non_pf_array_size) {
+        //        SCLogNotice("sgh:%p: %u pkt non-pf sigs, with alproto:%u, dsize:%u, dport:%u",
+        //        sgh, pkt_non_pf_array_size, nonpf_pkt_alproto, nonpf_pkt_dsize, nonpf_pkt_dport);
+        struct PrefilterNonPFData *data =
+                SCCalloc(1, sizeof(*data) + pkt_non_pf_array_size * sizeof(data->array[0]));
+        if (data == NULL)
+            goto error;
+        data->size = pkt_non_pf_array_size;
+        memcpy((uint8_t *)&data->array, pkt_non_pf_array,
+                pkt_non_pf_array_size * sizeof(data->array[0]));
+        // TODO can we set a mask here?
+        // TODO name
+        // TODO free func
+        if (PrefilterAppendEngine(
+                    de_ctx, sgh, PrefilterPktNonPF, 0, (void *)data, free, "pkt::non_pf") < 0) {
+            SCFree(data);
+            goto error;
+        }
+    }
+    if (frame_non_pf_array_size) {
+        SCLogDebug("%u frame non-pf sigs", frame_non_pf_array_size);
+        struct PrefilterNonPFData *data =
+                SCCalloc(1, sizeof(*data) + frame_non_pf_array_size * sizeof(data->array[0]));
+        if (data == NULL)
+            goto error;
+        data->size = frame_non_pf_array_size;
+        memcpy((uint8_t *)&data->array, frame_non_pf_array,
+                frame_non_pf_array_size * sizeof(data->array[0]));
+        // TODO free func
+        // TODO engine name
+        if (PrefilterAppendFrameEngine(de_ctx, sgh, PrefilterFrameNonPF, ALPROTO_UNKNOWN,
+                    FRAME_ANY_TYPE, (void *)data, free, "frame::non_pf") < 0) {
+            SCFree(data);
+            goto error;
+        }
+    }
+
+    SCFree(pkt_non_pf_array);
+    pkt_non_pf_array = NULL;
+    SCFree(frame_non_pf_array);
+    frame_non_pf_array = NULL;
+
     /* we have lists of engines in sgh->init now. Lets setup the
      * match arrays */
     PrefilterEngineList *el;
@@ -444,7 +853,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->pkt_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->pkt_engines == NULL) {
-            return;
+            goto error;
         }
         memset(sgh->pkt_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -469,7 +878,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->payload_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->payload_engines == NULL) {
-            return;
+            goto error;
         }
         memset(sgh->payload_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -494,7 +903,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->tx_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->tx_engines == NULL) {
-            return;
+            goto error;
         }
         memset(sgh->tx_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -570,7 +979,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->frame_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->frame_engines == NULL) {
-            return;
+            goto error;
         }
         memset(sgh->frame_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -589,6 +998,15 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             e++;
         }
     }
+    return 0;
+
+error:
+    if (tx_engines_hash) {
+        HashListTableFree(tx_engines_hash);
+    }
+    SCFree(pkt_non_pf_array);
+    SCFree(frame_non_pf_array);
+    return -1;
 }
 
 /* hash table for assigning a unique id to each engine type. */
diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h
index ec58594b9b00..04882ec633d7 100644
--- a/src/detect-engine-prefilter.h
+++ b/src/detect-engine-prefilter.h
@@ -72,7 +72,7 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
 
 void PrefilterFreeEnginesList(PrefilterEngineList *list);
 
-void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
+int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 
 #ifdef PROFILING
@@ -91,4 +91,5 @@ int PrefilterMultiGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
         const DetectBufferMpmRegistry *mpm_reg, int list_id);
 
+void PrefilterPktNonPFStatsDump(void);
 #endif
diff --git a/src/detect-engine-profile.c b/src/detect-engine-profile.c
index 36cf6e106b05..9f46fdac6ec2 100644
--- a/src/detect-engine-profile.c
+++ b/src/detect-engine-profile.c
@@ -47,7 +47,8 @@ void RulesDumpTxMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupH
     jb_set_uint(js, "rule_group_id", sgh->id);
     jb_set_uint(js, "rule_cnt", rule_cnt);
     jb_set_uint(js, "pkt_rule_cnt", pkt_prefilter_cnt);
-    jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt);
+    // TODO
+    // jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt);
 
     jb_open_array(js, "rules");
     for (uint32_t x = 0; x < rule_cnt; x++) {
@@ -91,7 +92,8 @@ void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx,
     jb_set_string(js, "inspect_type", "packet");
     jb_set_uint(js, "rule_group_id", sgh->id);
     jb_set_uint(js, "rule_cnt", det_ctx->match_array_cnt);
-    jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt);
+    // TODO
+    // jb_set_uint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt);
 
     jb_open_array(js, "rules");
     for (uint32_t x = 0; x < det_ctx->match_array_cnt; x++) {
diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index 49982f028b28..9c8364d403c3 100644
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -166,18 +166,6 @@ void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 
     SCLogDebug("sgh %p", sgh);
 
-    if (sgh->non_pf_other_store_array != NULL) {
-        SCFree(sgh->non_pf_other_store_array);
-        sgh->non_pf_other_store_array = NULL;
-        sgh->non_pf_other_store_cnt = 0;
-    }
-
-    if (sgh->non_pf_syn_store_array != NULL) {
-        SCFree(sgh->non_pf_syn_store_array);
-        sgh->non_pf_syn_store_array = NULL;
-        sgh->non_pf_syn_store_cnt = 0;
-    }
-
     if (sgh->init != NULL) {
         SigGroupHeadInitDataFree(sgh->init);
         sgh->init = NULL;
@@ -619,82 +607,6 @@ void SigGroupHeadSetupFiles(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     }
 }
 
-/** \brief build an array of rule id's for sigs with no prefilter
- *  Also updated de_ctx::non_pf_store_cnt_max to track the highest cnt
- */
-int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
-{
-    Signature *s = NULL;
-    uint32_t sig = 0;
-    uint32_t non_pf = 0;
-    uint32_t non_pf_syn = 0;
-
-    if (sgh == NULL)
-        return 0;
-
-    BUG_ON(sgh->non_pf_other_store_array != NULL);
-
-    for (sig = 0; sig < sgh->init->sig_cnt; sig++) {
-        s = sgh->init->match_array[sig];
-        if (s == NULL)
-            continue;
-
-        if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) {
-            if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
-                non_pf++;
-            }
-            non_pf_syn++;
-        }
-    }
-
-    if (non_pf == 0 && non_pf_syn == 0) {
-        sgh->non_pf_other_store_array = NULL;
-        sgh->non_pf_syn_store_array = NULL;
-        return 0;
-    }
-
-    if (non_pf > 0) {
-        sgh->non_pf_other_store_array = SCCalloc(non_pf, sizeof(SignatureNonPrefilterStore));
-        BUG_ON(sgh->non_pf_other_store_array == NULL);
-    }
-
-    if (non_pf_syn > 0) {
-        sgh->non_pf_syn_store_array = SCCalloc(non_pf_syn, sizeof(SignatureNonPrefilterStore));
-        BUG_ON(sgh->non_pf_syn_store_array == NULL);
-    }
-
-    for (sig = 0; sig < sgh->init->sig_cnt; sig++) {
-        s = sgh->init->match_array[sig];
-        if (s == NULL)
-            continue;
-
-        if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) {
-            if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
-                BUG_ON(sgh->non_pf_other_store_cnt >= non_pf);
-                BUG_ON(sgh->non_pf_other_store_array == NULL);
-                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].id = s->num;
-                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].mask = s->mask;
-                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].alproto = s->alproto;
-                sgh->non_pf_other_store_cnt++;
-            }
-
-            BUG_ON(sgh->non_pf_syn_store_cnt >= non_pf_syn);
-            BUG_ON(sgh->non_pf_syn_store_array == NULL);
-            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].id = s->num;
-            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].mask = s->mask;
-            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].alproto = s->alproto;
-            sgh->non_pf_syn_store_cnt++;
-        }
-    }
-
-    /* track highest cnt for any sgh in our de_ctx */
-    uint32_t max = MAX(sgh->non_pf_other_store_cnt, sgh->non_pf_syn_store_cnt);
-    if (max > de_ctx->non_pf_store_cnt_max)
-        de_ctx->non_pf_store_cnt_max = max;
-
-    return 0;
-}
-
 /**
  * \brief Check if a SigGroupHead contains a Signature, whose sid is sent as an
  *        argument.
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 60ad33ab3436..a4399aa74826 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -749,7 +749,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
     const int files_id = DetectBufferTypeGetByName("files");
     bool head_is_mpm = false;
     uint8_t last_id = DE_STATE_FLAG_BASE;
-    SCLogNotice("%u: setup app inspect engines. %u buffers", s->id, s->init_data->buffer_index);
+    SCLogDebug("%u: setup app inspect engines. %u buffers", s->id, s->init_data->buffer_index);
+    SCLogDebug("s->id %u mpm_list %d", s->id, mpm_list);
 
     for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
         SigMatchData *smd = SigMatchList2DataArray(s->init_data->buffers[x].head);
@@ -817,7 +818,7 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
             .progress = (uint16_t)s->init_data->hook.t.app.app_progress,
             .sm_list = (uint16_t)s->init_data->hook.sm_list,
             .sm_list_base = (uint16_t)s->init_data->hook.sm_list,
-            .dir = dir,
+            .dir = (uint8_t)dir,
         };
         AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm);
     }
@@ -2754,6 +2755,9 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
         SCDetectRequiresStatusFree(de_ctx->requirements);
     }
 
+    if (de_ctx->non_pf_engine_names) {
+        HashTableFree(de_ctx->non_pf_engine_names);
+    }
     SCFree(de_ctx);
     //DetectAddressGroupPrintMemory();
     //DetectSigGroupPrintMemory();
@@ -3320,13 +3324,6 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
         return TM_ECODE_FAILED;
     }
 
-    /* sized to the max of our sgh settings. A max setting of 0 implies that all
-     * sgh's have: sgh->non_pf_store_cnt == 0 */
-    if (de_ctx->non_pf_store_cnt_max > 0) {
-        det_ctx->non_pf_id_array =  SCCalloc(de_ctx->non_pf_store_cnt_max, sizeof(SigIntId));
-        BUG_ON(det_ctx->non_pf_id_array == NULL);
-    }
-
     /* DeState */
     if (de_ctx->sig_array_len > 0) {
         det_ctx->match_array_len = de_ctx->sig_array_len;
@@ -3579,10 +3576,6 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     if (det_ctx->spm_thread_ctx != NULL) {
         SpmDestroyThreadCtx(det_ctx->spm_thread_ctx);
     }
-
-    if (det_ctx->non_pf_id_array != NULL)
-        SCFree(det_ctx->non_pf_id_array);
-
     if (det_ctx->match_array != NULL)
         SCFree(det_ctx->match_array);
 
@@ -3633,7 +3626,7 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     }
 
     AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events);
-
+    PrefilterPktNonPFStatsDump();
     SCFree(det_ctx);
 
     ThresholdCacheThreadFree();
diff --git a/src/detect.c b/src/detect.c
index fba0fd2c5273..44999aa3ae06 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -71,7 +71,6 @@ typedef struct DetectRunScratchpad {
     const uint8_t flow_flags; /* flow/state flags: STREAM_* */
     const bool app_decoder_events;
     const SigGroupHead *sgh;
-    SignatureMask pkt_mask;
 } DetectRunScratchpad;
 
 /* prototypes */
@@ -264,118 +263,18 @@ const SigGroupHead *SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx,
     SCReturnPtr(sgh, "SigGroupHead");
 }
 
-static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
-                                            DetectEngineThreadCtx *det_ctx)
+static inline void DetectPrefilterCopyDeDup(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
 {
-    SigIntId mpm, nonmpm;
-    SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
-    SigIntId *nonmpm_ptr = det_ctx->non_pf_id_array;
-    uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
-    uint32_t n_cnt = det_ctx->non_pf_id_cnt;
-    SigIntId *final_ptr;
-    uint32_t final_cnt;
-    SigIntId id;
-    SigIntId previous_id = (SigIntId)-1;
+    SigIntId *pf_ptr = det_ctx->pmq.rule_id_array;
+    uint32_t final_cnt = det_ctx->pmq.rule_id_array_cnt;
     Signature **sig_array = de_ctx->sig_array;
     Signature **match_array = det_ctx->match_array;
-    Signature *s;
-
-    SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
-
-    /* Load first values. */
-    if (likely(m_cnt)) {
-        mpm = *mpm_ptr;
-    } else {
-        /* mpm list is empty */
-        final_ptr = nonmpm_ptr;
-        final_cnt = n_cnt;
-        goto final;
-    }
-    if (likely(n_cnt)) {
-        nonmpm = *nonmpm_ptr;
-    } else {
-        /* non-mpm list is empty. */
-        final_ptr = mpm_ptr;
-        final_cnt = m_cnt;
-        goto final;
-    }
-    while (1) {
-        if (mpm < nonmpm) {
-            /* Take from mpm list */
-            id = mpm;
-
-            s = sig_array[id];
-            /* As the mpm list can contain duplicates, check for that here. */
-            if (likely(id != previous_id)) {
-                *match_array++ = s;
-                previous_id = id;
-            }
-            if (unlikely(--m_cnt == 0)) {
-                /* mpm list is now empty */
-                final_ptr = nonmpm_ptr;
-                final_cnt = n_cnt;
-                goto final;
-             }
-             mpm_ptr++;
-             mpm = *mpm_ptr;
-         } else if (mpm > nonmpm) {
-             id = nonmpm;
-
-             s = sig_array[id];
-             /* As the mpm list can contain duplicates, check for that here. */
-             if (likely(id != previous_id)) {
-                 *match_array++ = s;
-                 previous_id = id;
-             }
-             if (unlikely(--n_cnt == 0)) {
-                 final_ptr = mpm_ptr;
-                 final_cnt = m_cnt;
-                 goto final;
-             }
-             nonmpm_ptr++;
-             nonmpm = *nonmpm_ptr;
-
-        } else { /* implied mpm == nonmpm */
-            /* special case: if on both lists, it's a negated mpm pattern */
-
-            /* mpm list may have dups, so skip past them here */
-            while (--m_cnt != 0) {
-                mpm_ptr++;
-                mpm = *mpm_ptr;
-                if (mpm != nonmpm)
-                    break;
-            }
-            /* if mpm is done, update nonmpm_ptrs and jump to final */
-            if (unlikely(m_cnt == 0)) {
-                n_cnt--;
-
-                /* mpm list is now empty */
-                final_ptr = ++nonmpm_ptr;
-                final_cnt = n_cnt;
-                goto final;
-            }
-            /* otherwise, if nonmpm is done jump to final for mpm
-             * mpm ptrs already updated */
-            if (unlikely(--n_cnt == 0)) {
-                final_ptr = mpm_ptr;
-                final_cnt = m_cnt;
-                goto final;
-            }
-
-            /* not at end of the lists, update nonmpm. Mpm already
-             * updated in while loop above. */
-            nonmpm_ptr++;
-            nonmpm = *nonmpm_ptr;
-        }
-    }
-
- final: /* Only one list remaining. Just walk that list. */
-
+    SigIntId previous_id = (SigIntId)-1;
     while (final_cnt-- > 0) {
-        id = *final_ptr++;
-        s = sig_array[id];
+        SigIntId id = *pf_ptr++;
+        Signature *s = sig_array[id];
 
-        /* As the mpm list can contain duplicates, check for that here. */
+        /* As the prefilter list can contain duplicates, check for that here. */
         if (likely(id != previous_id)) {
             *match_array++ = s;
             previous_id = id;
@@ -383,48 +282,10 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
     }
 
     det_ctx->match_array_cnt = match_array - det_ctx->match_array;
-    DEBUG_VALIDATE_BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_pf_id_cnt) < det_ctx->match_array_cnt);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->pmq.rule_id_array_cnt < det_ctx->match_array_cnt);
     PMQ_RESET(&det_ctx->pmq);
 }
 
-/** \internal
- *  \brief build non-prefilter list based on the rule group list we've set.
- */
-static inline void DetectPrefilterBuildNonPrefilterList(
-        DetectEngineThreadCtx *det_ctx, const SignatureMask mask, const AppProto alproto)
-{
-    for (uint32_t x = 0; x < det_ctx->non_pf_store_cnt; x++) {
-        /* only if the mask matches this rule can possibly match,
-         * so build the non_mpm array only for match candidates */
-        const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
-        const AppProto rule_alproto = det_ctx->non_pf_store_ptr[x].alproto;
-        if ((rule_mask & mask) == rule_mask &&
-                (rule_alproto == 0 || AppProtoEquals(rule_alproto, alproto))) {
-            det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id;
-        }
-    }
-}
-
-/** \internal
- *  \brief select non-mpm list
- *  Based on the packet properties, select the non-mpm list to use
- *  \todo move non_pf_store* into scratchpad */
-static inline void
-DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch)
-{
-    if ((p->proto == IPPROTO_TCP) && PacketIsTCP(p) && (PacketGetTCP(p)->th_flags & TH_SYN)) {
-        det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array;
-        det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt;
-    } else {
-        det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array;
-        det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt;
-    }
-    SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)",
-            det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt,
-            scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt,
-            scratch->sgh->non_pf_other_store_array, scratch->sgh->non_pf_other_store_cnt);
-}
-
 /** \internal
  *  \brief update flow's file tracking flags based on the detection engine
  *         A set of flags is prepared that is sent to the File API. The
@@ -676,42 +537,21 @@ static inline void DetectRunPrefilterPkt(
     DetectRunScratchpad *scratch
 )
 {
-    DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch);
-
     /* create our prefilter mask */
-    PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events);
-
-    /* build and prefilter non_pf list against the mask of the packet */
-    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST);
-    det_ctx->non_pf_id_cnt = 0;
-    if (likely(det_ctx->non_pf_store_cnt > 0)) {
-        DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto);
-    }
-    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_NONMPMLIST);
-
+    PacketCreateMask(p, &p->sig_mask, scratch->alproto, scratch->app_decoder_events);
     /* run the prefilter engines */
-    Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, scratch->pkt_mask);
+    Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, p->sig_mask);
     /* create match list if we have non-pf and/or pf */
-    if (det_ctx->non_pf_store_cnt || det_ctx->pmq.rule_id_array_cnt) {
+    if (det_ctx->pmq.rule_id_array_cnt) {
 #ifdef PROFILING
         if (tv) {
             StatsAddUI64(tv, det_ctx->counter_mpm_list, (uint64_t)det_ctx->pmq.rule_id_array_cnt);
         }
 #endif
         PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_SORT2);
-        DetectPrefilterMergeSort(de_ctx, det_ctx);
+        DetectPrefilterCopyDeDup(de_ctx, det_ctx);
         PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_SORT2);
     }
-
-#ifdef PROFILING
-    if (tv) {
-        StatsAddUI64(tv, det_ctx->counter_nonmpm_list,
-                             (uint64_t)det_ctx->non_pf_store_cnt);
-        /* non mpm sigs after mask prefilter */
-        StatsAddUI64(tv, det_ctx->counter_fnonmpm_list,
-                             (uint64_t)det_ctx->non_pf_id_cnt);
-    }
-#endif
 }
 
 /** \internal
@@ -808,8 +648,8 @@ static inline void DetectRulePacketRules(
 
         /* don't run mask check for stateful rules.
          * There we depend on prefilter */
-        if ((s->mask & scratch->pkt_mask) != s->mask) {
-            SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask);
+        if ((s->mask & p->sig_mask) != s->mask) {
+            SCLogDebug("mask mismatch %x & %x != %x", s->mask, p->sig_mask, s->mask);
             goto next;
         }
 
@@ -962,7 +802,7 @@ static DetectRunScratchpad DetectRunSetup(
         app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
     }
 
-    DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL, 0 };
+    DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL };
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_SETUP);
     return pad;
 }
@@ -1094,14 +934,14 @@ DetectRunTxSortHelper(const void *a, const void *b)
 void *DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto, uint8_t flow_flags)
 {
     if (unlikely(alproto == ALPROTO_DOH2)) {
-        if (engine_alproto == ALPROTO_DNS) {
+        if (engine_alproto == ALPROTO_DNS) { // || engine_alproto == ALPROTO_UNKNOWN) {
             // need to get the dns tx pointer
             tx_ptr = SCDoH2GetDnsTx(tx_ptr, flow_flags);
-        } else if (engine_alproto != ALPROTO_HTTP2) {
+        } else if (engine_alproto != ALPROTO_HTTP2 && engine_alproto != ALPROTO_UNKNOWN) {
             // incompatible engine->alproto with flow alproto
             tx_ptr = NULL;
         }
-    } else if (engine_alproto != alproto) {
+    } else if (engine_alproto != alproto && engine_alproto != ALPROTO_UNKNOWN) {
         // incompatible engine->alproto with flow alproto
         tx_ptr = NULL;
     }
diff --git a/src/detect.h b/src/detect.h
index 24b008adb2b0..6b8793b44bcb 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -309,9 +309,6 @@ typedef struct DetectPort_ {
 // vacancy 1x
 #define SIG_MASK_REQUIRE_ENGINE_EVENT       BIT_U8(7)
 
-/* for now a uint8_t is enough */
-#define SignatureMask uint8_t
-
 #define FILE_SIG_NEED_FILE          0x01
 #define FILE_SIG_NEED_FILENAME      0x02
 #define FILE_SIG_NEED_MAGIC         0x04    /**< need the start of the file */
@@ -896,10 +893,6 @@ typedef struct DetectEngineCtx_ {
 
     uint32_t signum;
 
-    /** Maximum value of all our sgh's non_mpm_store_cnt setting,
-     *  used to alloc det_ctx::non_mpm_id_array */
-    uint32_t non_pf_store_cnt_max;
-
     /* used by the signature ordering module */
     struct SCSigOrderFunc_ *sc_sig_order_funcs;
 
@@ -1084,6 +1077,8 @@ typedef struct DetectEngineCtx_ {
 
     /* number of signatures using filestore, limited as u16 */
     uint16_t filestore_cnt;
+
+    HashTable *non_pf_engine_names;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */
@@ -1139,11 +1134,6 @@ typedef struct DetectEngineThreadCtx_ {
     /* the thread to which this detection engine thread belongs */
     ThreadVars *tv;
 
-    /** Array of non-prefiltered sigs that need to be evaluated. Updated
-     *  per packet based on the rule group and traffic properties. */
-    SigIntId *non_pf_id_array;
-    uint32_t non_pf_id_cnt; // size is cnt * sizeof(uint32_t)
-
     uint32_t mt_det_ctxs_cnt;
     struct DetectEngineThreadCtx_ **mt_det_ctxs;
     HashTable *mt_det_ctxs_hash;
@@ -1231,9 +1221,6 @@ typedef struct DetectEngineThreadCtx_ {
     RuleMatchCandidateTx *tx_candidates;
     uint32_t tx_candidates_size;
 
-    SignatureNonPrefilterStore *non_pf_store_ptr;
-    uint32_t non_pf_store_cnt;
-
     MpmThreadCtx mtc; /**< thread ctx for the mpm */
     PrefilterRuleStore pmq;
 
@@ -1505,13 +1492,6 @@ typedef struct SigGroupHead_ {
 
     uint32_t id; /**< unique id used to index sgh_array for stats */
 
-    /* non prefilter list excluding SYN rules */
-    uint32_t non_pf_other_store_cnt;
-    uint32_t non_pf_syn_store_cnt;
-    SignatureNonPrefilterStore *non_pf_other_store_array; // size is non_mpm_store_cnt * sizeof(SignatureNonPrefilterStore)
-    /* non mpm list including SYN rules */
-    SignatureNonPrefilterStore *non_pf_syn_store_array; // size is non_mpm_syn_store_cnt * sizeof(SignatureNonPrefilterStore)
-
     PrefilterEngine *pkt_engines;
     PrefilterEngine *payload_engines;
     PrefilterEngine *tx_engines;
diff --git a/src/packet.c b/src/packet.c
index cb6dcf618380..076642ac6f38 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -97,6 +97,7 @@ void PacketReinit(Packet *p)
     p->recursion_level = 0;
     PACKET_FREE_EXTDATA(p);
     p->app_update_direction = 0;
+    p->sig_mask = 0;
     p->flags = 0;
     p->flowflags = 0;
     p->pkt_src = 0;
diff --git a/src/util-profiling-rulegroups.c b/src/util-profiling-rulegroups.c
index b25e89b01bb4..7e4612fbf0ca 100644
--- a/src/util-profiling-rulegroups.c
+++ b/src/util-profiling-rulegroups.c
@@ -263,13 +263,14 @@ SCProfilingSghUpdateCounter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *
     if (det_ctx != NULL && det_ctx->sgh_perf_data != NULL && sgh->id < det_ctx->de_ctx->sgh_array_cnt) {
         SCProfileSghData *p = &det_ctx->sgh_perf_data[sgh->id];
         p->checks++;
-
+#if 0 // TODO
         if (det_ctx->non_pf_store_cnt > 0) {
             if (det_ctx->non_pf_store_ptr == sgh->non_pf_syn_store_array)
                 p->non_mpm_syn++;
             else
                 p->non_mpm_generic++;
         }
+#endif
         p->post_prefilter_sigs_total += det_ctx->match_array_cnt;
         if (det_ctx->match_array_cnt > p->post_prefilter_sigs_max)
             p->post_prefilter_sigs_max = det_ctx->match_array_cnt;