diff --git a/doc/userguide/configuration/suricata-yaml.rst b/doc/userguide/configuration/suricata-yaml.rst index 8004ecce9167..77eaa32423bd 100644 --- a/doc/userguide/configuration/suricata-yaml.rst +++ b/doc/userguide/configuration/suricata-yaml.rst @@ -2636,6 +2636,8 @@ Engine analysis and profiling Suricata offers several ways of analyzing performance of rules and the engine itself. +.. _config:engine-analysis: + Engine-analysis ~~~~~~~~~~~~~~~ @@ -2871,6 +2873,24 @@ Using this default configuration, Teredo detection will run on UDP port 3544. If the `ports` parameter is missing, or set to `any`, all ports will be inspected for possible presence of Teredo. +Recursion Level +~~~~~~~~~~~~~~~ + +Flow matching via recursion level can be disabled. It is enabled by +default. + +:: + + decoder: + # Depending on packet pickup, incoming and outgoing tunnelled packets + # can be scanned before the kernel has stripped and encapsulated headers, + # respectively, leading to incoming and outgoing flows not being associated. + recursion-level: + use-for-tracking: true + +Using this default setting, flows will be associated only if the compared packet +headers are encapsulated in the same number of headers. + Advanced Options ---------------- diff --git a/doc/userguide/devguide/extending/app-layer/transactions.rst b/doc/userguide/devguide/extending/app-layer/transactions.rst index 1105aad97128..180f2d96540a 100644 --- a/doc/userguide/devguide/extending/app-layer/transactions.rst +++ b/doc/userguide/devguide/extending/app-layer/transactions.rst @@ -1,3 +1,5 @@ +.. _transactions: + ************ Transactions ************ diff --git a/doc/userguide/rules/index.rst b/doc/userguide/rules/index.rst index efe3d83137d8..57e9f20eb7ac 100644 --- a/doc/userguide/rules/index.rst +++ b/doc/userguide/rules/index.rst @@ -51,3 +51,4 @@ Suricata Rules tag vlan-keywords ldap-keywords + rule-types diff --git a/doc/userguide/rules/rule-types.rst b/doc/userguide/rules/rule-types.rst new file mode 100644 index 000000000000..7b8cfdd4351b --- /dev/null +++ b/doc/userguide/rules/rule-types.rst @@ -0,0 +1,1161 @@ +.. role:: example-rule-action +.. role:: example-rule-header +.. role:: example-rule-options +.. role:: example-rule-emphasis + +Rule Types and Categorization +============================= + +Once parsed, Suricata rules are categorized for performance and further +processing (as different rule types will be handled by specific engine modules). +The signature types are defined in `src/detect.h +`_: + +.. literalinclude:: ../../../src/detect.h + :caption: src/detect.h + :language: c + :start-after: // rule types documentation tag start: SignatureType + :end-before: // rule types documentation tag end: SignatureType + +In more human readable terms: + +.. list-table:: Suricata Rule Types, and their Engine Analysis Term + :header-rows: 1 + + * - Rule Type + - Code Symbol + - Engine-Analysis Representation + * - Decoder Events Only + - ``SIG_TYPE_DEONLY`` + - ``de_only`` + * - Packet + - ``SIG_TYPE_PKT`` + - ``pkt`` + * - IP Only + - ``SIG_TYPE_IPONLY`` + - ``ip_only`` + * - IP Only (contains negated address(es)) + - ``SIG_TYPE_LIKE_IPONLY`` + - ``like_ip_only`` + * - Protocol Detection Only + - ``SIG_TYPE_PDONLY`` + - ``pd_only`` + * - Packet-Stream + - ``SIG_TYPE_PKT_STREAM`` + - ``pkt_stream`` + * - Stream + - ``SIG_TYPE_STREAM`` + - ``stream`` + * - Application Layer Protocol + - ``SIG_TYPE_APPLAYER`` + - ``app_layer`` + * - Application Layer Protocol Transactions + - ``SIG_TYPE_APP_TX`` + - ``app_tx`` + +The rule type will impact: + + - To what does the signature action apply, in case of a match (`Action Scope`) + - When is the rule matched against traffic (`Inspection Hook`) + - Against what the rule matches (`Data Exposed`) + +This categorization is done taking into consideration the presence or absence of +certain rule elements, as well as the type of keywords used. The categorization +currently takes place in `src/detect-engine-build.c:void SignatureSetType() +`_. + +The ``SignatureSetType()`` overall flow is described below: + +.. image:: rule-types/OverallAlgoHorizontal.png + :align: center + :width: 600 + :alt: A flowchart representing the SignatureSetType function. + +Flowcharts expanding uncovered functions or portions of the overall algorithm +above are shown in the :ref:`detailed-flowcharts-sig-type` section. + +The following table lists all Suricata signature types, and how they impact the +aspects aforementioned. + +.. list-table:: Suricata Rule Types + :widths: 10 17 22 29 26 + :header-rows: 1 + + * - Type + - Action Scope + - Inspection Hook + - Data Exposed + - Keyword Examples + + (non-exhaustive) + * - :ref:`Decoder Events Only ` + + (``de_only``) + - Packet + - Per-broken/ invalid packet + - Decoding events + - ``decode-event`` + * - :ref:`Packet ` + + (``pkt``) + - Packet + - Per-packet basis + - Packet-level info (e.g.: header info) + - ``tcp-pkt``, ``itype``, ``tcp.hdr``, ``tcp.seq``, ``ttl`` etc. + * - :ref:`IP Only ` + + (``ip_only``) + - Flow (if existing). Packets (if not part of a flow) + - Once per direction + - IP addresses on the flow + - Source/ Destination field of a rule + * - :ref:`IP Only (contains negated address) ` :sup:`2` + + (``like_ip_only``) + - Flow + - All packets + - IP addresses on the flow + - Source/ Destination field of a rule containing negated address + * - :ref:`Protocol Detection Only ` + + (``pd_only``) + - Flow + - Once per direction, when protocol detection is done + - Protocol detected for the flow + - ``app-layer-protocol`` + * - :ref:`Packet-Stream ` + + (``pkt_stream``) + - Flow, if stateful :sup:`1` + - Per stream chunk, if stateful, per-packet if not + + (stream payload AND packet payload) + - The reassembled stream and/or payload data + - ``content`` with ``startswith`` or ``depth`` + * - :ref:`Stream ` + + (``stream``) + - Flow, if stateful :sup:`1` + - Stream chunks, if stateful, just packets if not + - Stream reassembled payload or packet payload data + - ``tcp-stream`` in protocol field; simple ``content``; ``byte_extract`` + * - :ref:`Application Layer Protocol ` + + (``app_layer``) + - Flow + - Per-packet basis + - 'protocol' field in a rule + - `Protocol field `_ of a rule + * - :ref:`Application Layer Protocol Transactions ` + + (``app_tx``) + - Flow + - Per :ref:`transaction ` update + - Buffer keywords + - Application layer protocol-related, e.g. ``http.host``, ``rfb.secresult``, + ``dcerpc.stub_data``, ``frame`` keywords + +.. note:: Action Scope: `Flow, if stateful` + + (1) Apply to the flow. If a segment isn't accepted into a stream for any + reason (such as packet anomalies, errors, memcap reached etc), the rule will + be applied on a packet level. + +.. warning:: + + Although both are related to matching on application layer protocols, as the + table suggests, since Suricata 7 a Protocol Detection rule (that uses the + ``app-layer-protocol`` keyword) is not internally classified the same as a + rule simply matching on the application layer protocol on the ``protocol`` + field. + +Signature Properties +-------------------- + +The `Action Scope` mentioned above relates to the Signature Properties, as seen in +`src/detect-engine.c `_: + +.. literalinclude:: ../../../src/detect-engine.c + :caption: src/detect-engine.c + :language: c + :start-after: // rule types documentation tag start: SignatureProperties + :end-before: // rule types documentation tag end: SignatureProperties + +Signature: Require Real Packet +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Aside from the scope of action of a signature, certain rule conditions will +require that it matches against a *real packet* (as opposed to a *pseudo packet*). +These rules are flagged with ``SIG_MASK_REQUIRE_REAL_PKT`` by the engine, and +will have ``real_pkt`` listed as one of the rule's ``requirements``. (See +``engine-analysis`` example output for the :ref:`pkt-rule-type` rule type.) + +A *pseudo packet* is an internal resource used by the engine when a flow is over +but there is still data to be processed, such as when there is a flow timeout. +A fake packet is then injected in the flow to finish up processing before ending it. + +Those two types will be more documented soon (tracking +`#7424 `_). + +.. _variable-like-keywords-sig-type: + +Signature Types and Variable-like Keywords +------------------------------------------ + +Keywords such as flow variables (``flowint``, ``flowbits``), ``datasets``, +and similar ones can alter the rule type, if present in a signature. + +That happens because the variable condition can change per packet. Thus, the +Signature is categorized as a `packet` rule. + +This affects rule types: + + - Application Layer (``app_layer``) + - Protocol Detection Only (``pd_only``) + - Decoder Events Only (``de_only``) + - IP Only (``ip_only``) :sup:`3` + - Like IP Only (``like_ip_only``) :sup:`3` + +The rule examples provided further cover some such cases, but the table below +lists those keywords with more details: + +.. list-table:: Variable-like Keywords + :header-rows: 1 + + * - Keyword + - Keyword Option + - Rule Type change? + * - ``flow`` + - ``to_server``, ``to_client`` + - no type changes :sup:`3` + * - ``flow`` + - ``established``, ``not_established`` + - to `packet` + * - ``flowbits``, ``xbits``, ``hostbits`` + - ``isset``, ``isnotset`` + - to `packet` + * - ``flowbits``, ``xbits``, ``hostbits`` + - ``set``, ``unset``, ``toggle`` + - no type change + * - ``flowint`` + - ``isset``, ``notset``, all operators + - to `packet` + * - ``flowint`` + - defining the variable; unseting; + - no type change + * - ``iprep`` + - ``isset``, ``notset``, all operators + - to `packet` + +.. note:: IP Only and Like IP Only + + (3) Unlike the other affected types, signatures that would otherwise be + classified as ``ip_only`` or ``like_ip_only`` become Packet rules if the + ``flow`` keyword is used, regardless of option. + +.. note:: + + ``dataset``, while may look similar to the keywords above, doesn't pertain + to this list as it can only be used with sticky buffer keywords, thus being + only available to Application Layer Transaction rules (`app_tx`), which are + not affected by this. + +Flowbits: ``isset`` +^^^^^^^^^^^^^^^^^^^ + +If a non-stateful rule (e.g. a ``pkt`` rule) checks if a flowbit is set (like in +*flowbits:fb6,isset*) and the rule that sets that variable is a stateful one, +such as an ``app_tx`` rule, the engine will set a flag to indicate that that +rule is also stateful - without altering its signature type. This flag is +currently ``SIG_FLAG_INIT_STATE_MATCH`` (cf. ticket `#7483 +`_). + +There is a work-in-progress to add information about this to the ``engine-analysis`` +report (ticket `#7456 `_). + + +Signatures per Type +------------------- + +This section offers brief descriptions for each rule type, and illustrates what +signatures of each type may look like. It is possible to learn the type of a +signature, as well as other important information, by running Suricata in +:ref:`engine analysis ` mode. + +For each rule type, there is also a sample of the Engine Analysis report +for one or more of rule(s) shown. + +.. _de-only-rule-type: + +Decoder Events Only +^^^^^^^^^^^^^^^^^^^ + +Signatures the inspect broken or invalid packets. They expose Suricata decoding +events. + +For more examples check https://github.com/OISF/suricata/blob/master/rules/decoder-events.rules. + +Example +""""""" + +.. container:: example-rule + + alert pkthdr any any -> any any (msg:"SURICATA IPv6 duplicated Hop-By-Hop Options extension header"; :example-rule-emphasis:`decode-event:ipv6.exthdr_dupl_hh;` classtype:protocol-command-decode; sid:1101;) + +.. container:: example-rule + + drop pkthdr any any -> any any (msg:"SURICATA IPv4 invalid option length"; `:example-rule-emphasis:`decode-event:ipv4.opt_invalid_len;` classtype:protocol-command-decode; sid:2200005; rev:2;) + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert pkthdr any any -> any any (msg:\"SURICATA IPv6 duplicated Hop-By-Hop Options extension header\"; decode-event:ipv6.exthdr_dupl_hh; classtype:protocol-command-decode; sid:1101;)", + "id": 1101, + "gid": 1, + "rev": 0, + "msg": "SURICATA IPv6 duplicated Hop-By-Hop Options extension header", + "app_proto": "unknown", + "requirements": [ + "engine_event" + ], + "type": "de_only", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "toserver", + "toclient" + ], + "pkt_engines": [ + { + "name": "packet", + "is_mpm": false + } + ], + "frame_engines": [], + "lists": { + "packet": { + "matches": [ + { + "name": "decode-event" + } + ] + } + } + } + + +.. _pkt-rule-type: + +Packet +^^^^^^ + +Rules that expose/ inspect information on a packet-level (for instance, the +header). Certain flow keywords may also turn a rule into a ``pkt`` rule, if +they require per-packet inspection (cf. :ref:`variable-like-keywords-sig-type`). + +Examples +"""""""" + +.. container:: example-rule + + alert :example-rule-emphasis:`tcp-pkt` any any -> any any (msg:"tcp-pkt, anchored content"; :example-rule-emphasis:`content:"abc"; startswith;` sid:203;) + +.. container:: example-rule + + alert tcp any any -> any any (msg:"ttl"; :example-rule-emphasis:`ttl:123;` sid:701;) + +.. container:: example-rule + + alert udp any any -> any any (msg:"UDP with flow direction"; flow:to_server; sid:1001;) + +.. container:: example-rule + + alert tcp any any -> any 443 (flow: to_server; flowbits:set,tls_error; sid:1604; msg:"Allow TLS error handling (outgoing packet) - non-stateful rule";) + +.. container:: example-rule + + alert tcp-pkt any any -> any any (msg:"Flowbit isset"; :example-rule-emphasis:`flowbits:isset,fb6; flowbits:isset,fb7;` sid:1919;) + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert tcp-pkt any any -> any any (msg:\"tcp-pkt, anchored content\"; content:\"abc\"; startswith; sid:203;)", + "id": 203, + "gid": 1, + "rev": 0, + "msg": "tcp-pkt, anchored content", + "app_proto": "unknown", + "requirements": [ + "payload", + "real_pkt" + ], + "type": "pkt", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "need_packet", + "toserver", + "toclient", + "prefilter" + ], + "pkt_engines": [ + { + "name": "payload", + "is_mpm": true + } + ], + "frame_engines": [], + "lists": { + "payload": { + "matches": [ + { + "name": "content", + "content": { + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": true, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "depth": 3, + "fast_pattern": false, + "relative_next": false + } + } + ] + } + }, + "mpm": { + "buffer": "payload", + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": true, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "depth": 3, + "fast_pattern": false, + "relative_next": false + } + } + +.. _ip-only-rule-type: + +IP Only +^^^^^^^ + +The IP ONLY rule type is used when rules match only on source and destination +IP addresses, and not on any other flow or content modifier. + +Examples +"""""""" + +.. container:: example-rule + + alert tcp-stream :example-rule-emphasis:`any` any -> :example-rule-emphasis:`any` any (msg:"tcp-stream, no content"; sid:101;) + +.. container:: example-rule + + alert tcp-pkt :example-rule-emphasis:`[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]` any -> :example-rule-emphasis:`any` any (msg:"tcp-pkt, no content"; sid:201;) + +.. container:: example-rule + + alert ip :example-rule-emphasis:`any` any -> :example-rule-emphasis:`any` any (:example-rule-emphasis:`hostbits:set,myflow2;` sid:1505;) + +.. container:: example-rule + + alert udp :example-rule-emphasis:`any` any -> :example-rule-emphasis:`any` any (msg:"UDP with flow direction"; sid:1601;) + + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert ip any any -> any any (hostbits:set,myflow2; sid:1505;)", + "id": 1505, + "gid": 1, + "rev": 0, + "app_proto": "unknown", + "requirements": [], + "type": "ip_only", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "toserver", + "toclient" + ], + "pkt_engines": [], + "frame_engines": [], + "lists": { + "postmatch": { + "matches": [ + { + "name": "hostbits" + } + ] + } + } + } + +.. _like-ip-only-rule-type: + +IP Only (contains negated address) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A rule that inspects IP only properties, but contains negated IP addresses. + +IP Only signatures with negated addresses are `like` IP-only signatures, but +currently handled differently due to limitations of the algorithm processing +IP Only rules. Impactful differences from a user-perspective are listed on the +Signature Types table. + +Examples +"""""""" + +.. container:: example-rule + + alert tcp 192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 any -> :example-rule-emphasis:`![192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]` any (msg:"tcp, has negated IP address"; sid:304;) + +.. container:: example-rule + + alert tcp :example-rule-emphasis:`[10.0.0.0/8,!10.10.10.10]` any -> :example-rule-emphasis:`[10.0.0.0/8,!10.10.10.10]` any (msg:"tcp, has negated IP address"; sid:305;) + + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert tcp [10.0.0.0/8,!10.10.10.10] any -> [10.0.0.0/8,!10.10.10.10] any (msg:\"tcp, has negated IP address\"; sid:305;)", + "id": 305, + "gid": 1, + "rev": 0, + "msg": "tcp, has negated IP address", + "app_proto": "unknown", + "requirements": [], + "type": "like_ip_only", + "flags": [ + "sp_any", + "dp_any", + "toserver", + "toclient" + ], + "pkt_engines": [], + "frame_engines": [], + "lists": {} + } + +.. _pd-only-rule-type: + +Protocol Detection Only +^^^^^^^^^^^^^^^^^^^^^^^ + +When a signature checks for the application layer protocol but there is no need +for a per-packet inspection, protocol detection can be done with the +``app-layer-protocol`` keyword. Check the `keyword documentation +`_ +full for usage. + +See :ref:`Protocol Detection Only ` for a flowchart +representing how the type is defined. + +See :ref:`app-layer-rule-type` for a packet-based inspection. + +.. warning:: + + Since Suricata 7, a Protocol Detection rule (that uses the + ``app-layer-protocol`` keyword) is not internally classified the same as a + rule simply matching on the application layer protocol on the ``protocol`` + field. + +Examples +"""""""" + +.. container:: example-rule + + alert tcp any any -> any any (msg:"tcp, pd negated"; :example-rule-emphasis:`app-layer-protocol:!http;` sid:401;) + +.. container:: example-rule + + alert tcp any any -> any any (msg:"tcp, pd positive"; :example-rule-emphasis:`app-layer-protocol:http;` sid:402;) + +.. container:: example-rule + + alert tcp any any -> any any (msg:"tcp, pd positive dns"; :example-rule-emphasis:`app-layer-protocol:dns;` sid:403;) + +.. container:: example-rule + + alert tcp any any -> any any (msg:"tcp, pd positive, dns, flow:to_server"; :example-rule-emphasis:`app-layer-protocol:dns;` flow:to_server; sid:405;) + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert tcp any any -> any any (msg:\"tcp, pd positive dns\"; app-layer-protocol:dns; sid:403;)", + "id": 403, + "gid": 1, + "rev": 0, + "msg": "tcp, pd positive dns", + "app_proto": "unknown", + "requirements": [], + "type": "pd_only", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "toserver", + "toclient" + ], + "pkt_engines": [ + { + "name": "packet", + "is_mpm": false + } + ], + "frame_engines": [], + "lists": { + "packet": { + "matches": [ + { + "name": "app-layer-protocol" + } + ] + } + } + } + +.. _pkt-stream-rule-type: + +Packet-Stream +^^^^^^^^^^^^^ + +A rule is categorized as such when it inspects on traffic in specific portions +of the packet payload, using ``content`` buffer with the ``startswith`` or +``depth`` keywords. + +Examples +"""""""" + +.. container:: example-rule + + alert tcp any any -> any any (msg:"tcp, anchored content"; :example-rule-emphasis:`content:"abc"; startswith;` sid:303;) + +.. container:: example-rule + + alert http any any -> any any (msg:"http, anchored content"; :example-rule-emphasis:`content:"abc"; depth:30;` sid:603;) + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert http any any -> any any (msg:\"http, anchored content\"; content:\"abc\"; depth:30; sid:603;)", + "id": 603, + "gid": 1, + "rev": 0, + "msg": "http, anchored content", + "app_proto": "http_any", + "requirements": [ + "payload", + "flow" + ], + "type": "pkt_stream", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "applayer", + "need_packet", + "need_stream", + "toserver", + "toclient", + "prefilter" + ], + "pkt_engines": [ + { + "name": "payload", + "is_mpm": true + } + ], + "frame_engines": [], + "lists": { + "payload": { + "matches": [ + { + "name": "content", + "content": { + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "depth": 30, + "fast_pattern": false, + "relative_next": false + } + } + ] + } + }, + "mpm": { + "buffer": "payload", + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "depth": 30, + "fast_pattern": false, + "relative_next": false + } + } + +.. _stream-rule-type: + +Stream +^^^^^^ + +A rule that matches payload traffic without regards to its position, that is, +on an unanchored ``content`` buffer, uses byte extraction or matches on +``tcp-stream`` is classified a stream rule. + +Examples +"""""""" + +.. container:: example-rule + + alert :example-rule-emphasis:`tcp-stream` any any -> any any (msg:"tcp-stream, simple content"; :example-rule-emphasis:`content:"abc";` sid:102;) + +.. container:: example-rule + + alert :example-rule-emphasis:`http` any any -> any any (msg:"http, simple content"; :example-rule-emphasis:`content:"abc";` sid:602;) + +.. container:: example-rule + + alert tcp any any -> any 443 (:example-rule-emphasis:`flow: to_server; content:"abc";` flowbits:set,tls_error; sid:1605; msg:"Allow TLS error handling (outgoing packet) with simple content - Stream rule";) + +.. container:: example-rule + + alert tcp any any -> any 443 (:example-rule-emphasis:`flow: to_server; content:"abc";` sid:160401; msg:"Allow TLS error handling (outgoing packet) - stream rule";) + +.. container:: example-rule + + alert tcp any any -> any 443 (:example-rule-emphasis:`content:"abc";` sid:160402; msg:"Allow TLS error handling (outgoing packet) - stream rule";) + +.. container:: example-rule + + alert :example-rule-emphasis:`tcp` any any -> any any (msg:"byte_extract with dce"; :example-rule-emphasis:`byte_extract:4,0,var,dce; byte_test:4,>,var,4,little;` sid:901;) + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert tcp any any -> any any (msg:\"byte_extract with dce\"; byte_extract:4,0,var,dce; byte_test:4,>,var,4,little; sid:901;)", + "id": 901, + "gid": 1, + "rev": 0, + "msg": "byte_extract with dce", + "app_proto": "dcerpc", + "requirements": [ + "payload", + "flow" + ], + "type": "stream", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "applayer", + "need_stream", + "toserver", + "toclient" + ], + "pkt_engines": [ + { + "name": "payload", + "is_mpm": false + } + ], + "frame_engines": [], + "lists": { + "payload": { + "matches": [ + { + "name": "byte_extract" + }, + { + "name": "byte_test", + "byte_test": { + "nbytes": 4, + "offset": 4, + "base": "unset", + "flags": [ + "little_endian" + ] + } + } + ] + } + } + } + +.. _app-layer-rule-type: + +Application Layer Protocol +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For a packet-based inspection of the application layer protocol, a rule should +use the `protocol `_ field for the matches. + +.. warning:: + + Since Suricata 7, a simple rule matching traffic on the ``protocol`` field + is not internally classified the same as a rule using the ``app-layer-protocol`` + keyword). + +.. warning:: + + As per Suricata 7, if ``flow:established`` or ``flow:not_established`` is added + to a base Application Layer Protocol rule, that signature will become a + :ref:`pkt-rule-type` rule. + +Examples +"""""""" + +.. container:: example-rule + + alert :example-rule-emphasis:`dns` any any -> any any (msg:"app-layer, dns"; sid:404;) + +.. container:: example-rule + + alert :example-rule-emphasis:`http` any any -> any any (msg:"http, no content"; sid:601;) + +.. container:: example-rule + + alert :example-rule-emphasis:`tls` any any -> any any (msg:"tls, pkt or app-layer?"; flowint:tls_error_int,=,0; sid:613;) + + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert dns any any -> any any (msg:\"app-layer, dns\"; sid:404;)", + "id": 404, + "gid": 1, + "rev": 0, + "msg": "app-layer, dns", + "app_proto": "dns", + "requirements": [ + "flow" + ], + "type": "app_layer", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "applayer", + "toserver", + "toclient" + ], + "pkt_engines": [], + "frame_engines": [], + "lists": {} + } + +.. _app-tx-rule-type: + +Application Layer Protocol Transactions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rules inspecting traffic using keywords related to application layer protocols +are classified with this signature type. This also includes `frame` keywords. + +Examples +"""""""" + +.. container:: example-rule + + alert tcp any any -> any any (msg:"http, pos event"; :example-rule-emphasis:`app-layer-event:http.file_name_too_long;` sid:501;) + +.. container:: example-rule + + alert http any any -> any any (msg:"Test"; flow:established,to_server; :example-rule-emphasis:`http.method; content:"GET"; http.uri; content:".exe";` endswith; :example-rule-emphasis:`http.host; content:!".google.com";` endswith; sid:1102;) + +.. container:: example-rule + + alert udp any any -> any any (msg:"DNS UDP Frame"; flow:to_server; :example-rule-emphasis:`frame:dns.pdu;` content:"\|01 20 00 01\|"; offset:2; content:"suricata"; offset:13; sid:1402; rev:1;) + +.. container:: example-rule + + alert tcp any any -> any any (msg:"byte_extract with dce"; :example-rule-emphasis:`dcerpc.stub_data;` content:"abc"; byte_extract:4,0,var,relative; byte_test:4,>,var,4,little; sid:902;) + +Engine-Analysis Report +"""""""""""""""""""""" +.. code-block:: json + + { + "raw": "alert tcp any any -> any any (msg:\"byte_extract with dce\"; dcerpc.stub_data; content:\"abc\"; byte_extract:4,0,var,relative; byte_test:4,>,var,4,little; sid:902;)", + "id": 902, + "gid": 1, + "rev": 0, + "msg": "byte_extract with dce", + "app_proto": "dcerpc", + "requirements": [ + "flow" + ], + "type": "app_tx", + "flags": [ + "src_any", + "dst_any", + "sp_any", + "dp_any", + "applayer", + "toserver", + "toclient", + "prefilter" + ], + "pkt_engines": [], + "frame_engines": [], + "engines": [ + { + "name": "dce_stub_data", + "direction": "toclient", + "is_mpm": true, + "app_proto": "dcerpc", + "progress": 0, + "matches": [ + { + "name": "content", + "content": { + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "fast_pattern": false, + "relative_next": true + } + }, + { + "name": "byte_extract" + }, + { + "name": "byte_test", + "byte_test": { + "nbytes": 4, + "offset": 4, + "base": "unset", + "flags": [ + "little_endian" + ] + } + } + ] + }, + { + "name": "dce_stub_data", + "direction": "toserver", + "is_mpm": true, + "app_proto": "dcerpc", + "progress": 0, + "matches": [ + { + "name": "content", + "content": { + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "fast_pattern": false, + "relative_next": true + } + }, + { + "name": "byte_extract" + }, + { + "name": "byte_test", + "byte_test": { + "nbytes": 4, + "offset": 4, + "base": "unset", + "flags": [ + "little_endian" + ] + } + } + ] + }, + { + "name": "dce_stub_data", + "direction": "toclient", + "is_mpm": true, + "app_proto": "smb", + "progress": 0, + "matches": [ + { + "name": "content", + "content": { + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "fast_pattern": false, + "relative_next": true + } + }, + { + "name": "byte_extract" + }, + { + "name": "byte_test", + "byte_test": { + "nbytes": 4, + "offset": 4, + "base": "unset", + "flags": [ + "little_endian" + ] + } + } + ] + }, + { + "name": "dce_stub_data", + "direction": "toserver", + "is_mpm": true, + "app_proto": "smb", + "progress": 0, + "matches": [ + { + "name": "content", + "content": { + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "fast_pattern": false, + "relative_next": true + } + }, + { + "name": "byte_extract" + }, + { + "name": "byte_test", + "byte_test": { + "nbytes": 4, + "offset": 4, + "base": "unset", + "flags": [ + "little_endian" + ] + } + } + ] + } + ], + "lists": {}, + "mpm": { + "buffer": "dce_stub_data", + "pattern": "abc", + "length": 3, + "nocase": false, + "negated": false, + "starts_with": false, + "ends_with": false, + "is_mpm": true, + "no_double_inspect": false, + "fast_pattern": false, + "relative_next": true + } + } + +.. _detailed-flowcharts-sig-type: + +Detailed Flowcharts +------------------- + +A look into the illustrated overall representation of functions or paths that +determine signature types. + +.. _flowchart-ip-only-sig-type: + +IP Only and IP Only with negated addresses +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``ip_only`` and ``like_ip_only`` flows. + +.. image:: rule-types/IP-Only.png + :align: center + :alt: A flowchart representing the SignatureIsIPOnly function. + +.. _flowchart-pd-only-sig-type: + +Protocol Detection Only +^^^^^^^^^^^^^^^^^^^^^^^ + +``pd_only`` flow. + +.. image:: rule-types/PD-only.png + :align: center + :width: 400 + :alt: A flowchart representing the SignatureIsPDOnly function. + +.. _flowchart-app-layer-packet-app-tx-stream-sig-types: + +Application Layer Protocol, Transaction, Packet, Stream and Stream-Packet rules +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``app_layer``, ``app_tx``, ``pkt``, ``stream`` and ``stream-pkt`` flows. + +``REQUIRE_PACKET_`` and ``REQUIRE_STREAM`` can be seen as flags ``need_packet`` +and ``need_stream`` in the ``engine-analysis`` output. + +.. image:: rule-types/APP_Layer-Packet-TX-Stream.png + :align: center + :alt: A flowchart representing the portion of SignatureSetType function + that handles app_layer, app_tx, stream, pkt_stream and pkt rules. diff --git a/doc/userguide/rules/rule-types/APP_Layer-Packet-TX-Stream.png b/doc/userguide/rules/rule-types/APP_Layer-Packet-TX-Stream.png new file mode 100644 index 000000000000..0be76dd250f0 Binary files /dev/null and b/doc/userguide/rules/rule-types/APP_Layer-Packet-TX-Stream.png differ diff --git a/doc/userguide/rules/rule-types/IP-Only.png b/doc/userguide/rules/rule-types/IP-Only.png new file mode 100644 index 000000000000..b19c075b425b Binary files /dev/null and b/doc/userguide/rules/rule-types/IP-Only.png differ diff --git a/doc/userguide/rules/rule-types/OverallAlgoHorizontal.png b/doc/userguide/rules/rule-types/OverallAlgoHorizontal.png new file mode 100644 index 000000000000..c273a1380c63 Binary files /dev/null and b/doc/userguide/rules/rule-types/OverallAlgoHorizontal.png differ diff --git a/doc/userguide/rules/rule-types/PD-only.png b/doc/userguide/rules/rule-types/PD-only.png new file mode 100644 index 000000000000..75e81d80d789 Binary files /dev/null and b/doc/userguide/rules/rule-types/PD-only.png differ diff --git a/doc/userguide/rules/rule-types/RawFlowcharts/APP_Layer-Packet-TX-Stream.drawio b/doc/userguide/rules/rule-types/RawFlowcharts/APP_Layer-Packet-TX-Stream.drawio new file mode 100644 index 000000000000..e3a4b746f08e --- /dev/null +++ b/doc/userguide/rules/rule-types/RawFlowcharts/APP_Layer-Packet-TX-Stream.drawio @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/userguide/rules/rule-types/RawFlowcharts/IP-Only.drawio b/doc/userguide/rules/rule-types/RawFlowcharts/IP-Only.drawio new file mode 100644 index 000000000000..56d75d67d660 --- /dev/null +++ b/doc/userguide/rules/rule-types/RawFlowcharts/IP-Only.drawio @@ -0,0 +1,305 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/userguide/rules/rule-types/RawFlowcharts/OverallAlgoHorizontal.drawio b/doc/userguide/rules/rule-types/RawFlowcharts/OverallAlgoHorizontal.drawio new file mode 100644 index 000000000000..b1065278a710 --- /dev/null +++ b/doc/userguide/rules/rule-types/RawFlowcharts/OverallAlgoHorizontal.drawio @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/userguide/rules/rule-types/RawFlowcharts/PD-only.drawio b/doc/userguide/rules/rule-types/RawFlowcharts/PD-only.drawio new file mode 100644 index 000000000000..956f94dfd570 --- /dev/null +++ b/doc/userguide/rules/rule-types/RawFlowcharts/PD-only.drawio @@ -0,0 +1,225 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index 03f9a405816a..75ef52c53056 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -91,9 +91,9 @@ int HTPFileOpen(HtpState *s, HtpTxUserData *tx, const uint8_t *filename, uint16_ * @param[in] rawvalue * @param[out] range * - * @return HTP_OK on success, HTP_ERROR on failure. + * @return HTP_STATUS_OK on success, HTP_STATUS_ERROR on failure. */ -int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range) +int HTPParseContentRange(const bstr *rawvalue, HTTPContentRange *range) { uint32_t len = (uint32_t)bstr_len(rawvalue); return rs_http_parse_content_range(range, bstr_ptr(rawvalue), len); @@ -105,10 +105,10 @@ int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range) * @param[in] rawvalue * @param[out] range * - * @return HTP_OK on success, HTP_ERROR, -2, -3 on failure. + * @return HTP_STATUS_OK on success, HTP_STATUS_ERROR, -2, -3 on failure. */ static int HTPParseAndCheckContentRange( - bstr *rawvalue, HTTPContentRange *range, HtpState *s, HtpTxUserData *htud) + const bstr *rawvalue, HTTPContentRange *range, HtpState *s, HtpTxUserData *htud) { int r = HTPParseContentRange(rawvalue, range); if (r != 0) { @@ -147,8 +147,8 @@ static int HTPParseAndCheckContentRange( * \retval -1 error */ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filename, - uint16_t filename_len, const uint8_t *data, uint32_t data_len, htp_tx_t *tx, bstr *rawvalue, - HtpTxUserData *htud) + uint16_t filename_len, const uint8_t *data, uint32_t data_len, const htp_tx_t *tx, + const bstr *rawvalue, HtpTxUserData *htud) { SCEnter(); uint16_t flags; @@ -183,7 +183,7 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena uint32_t keylen; if (htp_tx_request_hostname(tx) != NULL) { uint32_t hlen = (uint32_t)bstr_len(htp_tx_request_hostname(tx)); - if (bstr_len(htp_tx_request_hostname(tx)) > UINT16_MAX) { + if (hlen > UINT16_MAX) { hlen = UINT16_MAX; } keylen = hlen + filename_len; diff --git a/src/app-layer-htp-file.h b/src/app-layer-htp-file.h index 53a47d5d3020..1fac823ce423 100644 --- a/src/app-layer-htp-file.h +++ b/src/app-layer-htp-file.h @@ -30,12 +30,12 @@ int HTPFileOpen( HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t, uint8_t); int HTPFileOpenWithRange(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, - uint32_t, htp_tx_t *, bstr *rawvalue, HtpTxUserData *htud); + uint32_t, const htp_tx_t *, const bstr *rawvalue, HtpTxUserData *htud); bool HTPFileCloseHandleRange(const StreamingBufferConfig *sbcfg, FileContainer *, const uint16_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t); int HTPFileStoreChunk(HtpTxUserData *, const uint8_t *, uint32_t, uint8_t); -int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range); +int HTPParseContentRange(const bstr *rawvalue, HTTPContentRange *range); int HTPFileClose(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t direction); diff --git a/src/app-layer-htp-libhtp.h b/src/app-layer-htp-libhtp.h index 726b5e0e0783..8ae6dcccef73 100644 --- a/src/app-layer-htp-libhtp.h +++ b/src/app-layer-htp-libhtp.h @@ -132,6 +132,13 @@ #define htp_header_value_ptr(h) bstr_ptr(h->value) #define htp_header_value(h) h->value +// Functions introduced to handle opaque htp_headers_t: +#define htp_headers_size(headers) htp_table_size(headers) +#define htp_headers_get_index(headers, index) htp_table_get_index(headers, index, NULL) +#define htp_tx_request_headers_size(tx) htp_table_size(tx->request_headers) +#define htp_tx_request_header_index(tx, i) htp_table_get_index(tx->request_headers, i, NULL); +#define htp_headers_t htp_table_t + bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri, bool uri_include_all); #endif /* SURICATA_APP_LAYER_HTP_LIBHTP__H */ diff --git a/src/app-layer-htp-xff.c b/src/app-layer-htp-xff.c index 0fe5ebc9c65a..a4096f0c8ee4 100644 --- a/src/app-layer-htp-xff.c +++ b/src/app-layer-htp-xff.c @@ -139,20 +139,17 @@ int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, return 0; } - htp_header_t *h_xff = NULL; - if (htp_tx_request_headers(tx) != NULL) { - h_xff = htp_tx_request_header(tx, xff_cfg->header); - } + const htp_header_t *h_xff = htp_tx_request_header(tx, xff_cfg->header); - if (h_xff != NULL && bstr_len(h_xff->value) >= XFF_CHAIN_MINLEN && - bstr_len(h_xff->value) < XFF_CHAIN_MAXLEN) { + if (h_xff != NULL && htp_header_value_len(h_xff) >= XFF_CHAIN_MINLEN && + htp_header_value_len(h_xff) < XFF_CHAIN_MAXLEN) { - memcpy(xff_chain, bstr_ptr(h_xff->value), bstr_len(h_xff->value)); - xff_chain[bstr_len(h_xff->value)]=0; + memcpy(xff_chain, htp_header_value_ptr(h_xff), htp_header_value_len(h_xff)); + xff_chain[htp_header_value_len(h_xff)] = 0; if (xff_cfg->flags & XFF_REVERSE) { /** Get the last IP address from the chain */ - p_xff = memrchr(xff_chain, ' ', bstr_len(h_xff->value)); + p_xff = memrchr(xff_chain, ' ', htp_header_value_len(h_xff)); if (p_xff == NULL) { p_xff = xff_chain; } else { @@ -161,7 +158,7 @@ int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, } else { /** Get the first IP address from the chain */ - p_xff = memchr(xff_chain, ',', bstr_len(h_xff->value)); + p_xff = memchr(xff_chain, ',', htp_header_value_len(h_xff)); if (p_xff != NULL) { *p_xff = 0; } diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 5b514cb45065..1299f607ec1f 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -2942,7 +2942,7 @@ static int HTPParserTest01(void) htp_tx_t *tx = HTPStateGetTx(htp_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(strcmp(bstr_util_strdup_to_c(htp_header_value(h)), "Victor/1.0")); @@ -2986,7 +2986,7 @@ static int HTPParserTest01b(void) htp_tx_t *tx = HTPStateGetTx(htp_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(strcmp(bstr_util_strdup_to_c(htp_header_value(h)), "Victor/1.0")); @@ -3041,7 +3041,7 @@ static int HTPParserTest01c(void) htp_tx_t *tx = HTPStateGetTx(htp_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(strcmp(bstr_util_strdup_to_c(htp_header_value(h)), "Victor/1.0")); @@ -3097,7 +3097,7 @@ static int HTPParserTest01a(void) htp_tx_t *tx = HTPStateGetTx(htp_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(strcmp(bstr_util_strdup_to_c(htp_header_value(h)), "Victor/1.0")); @@ -3139,7 +3139,7 @@ static int HTPParserTest02(void) htp_tx_t *tx = HTPStateGetTx(http_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NOT_NULL(h); FAIL_IF_NULL(htp_tx_request_method(tx)); @@ -3193,7 +3193,7 @@ static int HTPParserTest03(void) htp_tx_t *tx = HTPStateGetTx(htp_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NOT_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_UNKNOWN); FAIL_IF(htp_tx_request_protocol_number(tx) != HTP_PROTOCOL_V1_0); @@ -3234,7 +3234,7 @@ static int HTPParserTest04(void) htp_tx_t *tx = HTPStateGetTx(htp_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NOT_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_UNKNOWN); @@ -3308,7 +3308,7 @@ static int HTPParserTest05(void) FAIL_IF_NOT(htp_tx_request_method_number(tx) == HTP_METHOD_POST); FAIL_IF_NOT(htp_tx_request_protocol_number(tx) == HTP_PROTOCOL_V1_0); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF_NOT(htp_tx_response_status_number(tx) == 200); @@ -3399,7 +3399,7 @@ static int HTPParserTest06(void) FAIL_IF(htp_tx_response_status_number(tx) != 200); FAIL_IF(htp_tx_request_protocol_number(tx) != HTP_PROTOCOL_V1_1); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); AppLayerParserThreadCtxFree(alp_tctx); @@ -3638,7 +3638,7 @@ static int HTPParserTest10(void) FAIL_IF_NULL(htp_state); htp_tx_t *tx = HTPStateGetTx(htp_state, 0); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); char *name = bstr_util_strdup_to_c(htp_header_name(h)); @@ -3816,7 +3816,7 @@ static int HTPParserTest13(void) htp_state = f->alstate; FAIL_IF_NULL(htp_state); htp_tx_t *tx = HTPStateGetTx(htp_state, 0); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); char *name = bstr_util_strdup_to_c(htp_header_name(h)); @@ -5389,7 +5389,7 @@ static int HTPParserTest20(void) FAIL_IF_NULL(http_state); htp_tx_t *tx = HTPStateGetTx(http_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_GET); @@ -5448,7 +5448,7 @@ static int HTPParserTest21(void) FAIL_IF_NULL(http_state); htp_tx_t *tx = HTPStateGetTx(http_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_GET); @@ -5502,7 +5502,7 @@ static int HTPParserTest22(void) FAIL_IF_NULL(http_state); htp_tx_t *tx = HTPStateGetTx(http_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_GET); @@ -5556,7 +5556,7 @@ static int HTPParserTest23(void) FAIL_IF_NULL(http_state); htp_tx_t *tx = HTPStateGetTx(http_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_GET); @@ -5610,7 +5610,7 @@ static int HTPParserTest24(void) FAIL_IF_NULL(http_state); htp_tx_t *tx = HTPStateGetTx(http_state, 0); FAIL_IF_NULL(tx); - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(tx), 0, NULL); + htp_header_t *h = htp_tx_request_header_index(tx, 0); FAIL_IF_NULL(h); FAIL_IF(htp_tx_request_method_number(tx) != HTP_METHOD_GET); diff --git a/src/app-layer-http2.c b/src/app-layer-http2.c index f214d2210d23..bababc099c22 100644 --- a/src/app-layer-http2.c +++ b/src/app-layer-http2.c @@ -90,9 +90,9 @@ void HTTP2MimicHttp1Request(void *alstate_orig, void *h2s) rs_http2_tx_set_uri(h2s, bstr_ptr(htp_tx_request_uri(h1tx)), (uint32_t)bstr_len(htp_tx_request_uri(h1tx))); } - size_t nbheaders = htp_table_size(htp_tx_request_headers(h1tx)); + size_t nbheaders = htp_tx_request_headers_size(h1tx); for (size_t i = 0; i < nbheaders; i++) { - htp_header_t *h = htp_table_get_index(htp_tx_request_headers(h1tx), i, NULL); + const htp_header_t *h = htp_tx_request_header_index(h1tx, i); rs_http2_tx_add_header(h2s, htp_header_name_ptr(h), (uint32_t)htp_header_name_len(h), htp_header_value_ptr(h), (uint32_t)htp_header_value_len(h)); } diff --git a/src/detect-engine.c b/src/detect-engine.c index d1429f19f3c0..d576e550138a 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -110,6 +110,7 @@ static DetectEnginePktInspectionEngine *g_pkt_inspect_engines = NULL; static DetectEngineFrameInspectionEngine *g_frame_inspect_engines = NULL; // clang-format off +// rule types documentation tag start: SignatureProperties const struct SignatureProperties signature_properties[SIG_TYPE_MAX] = { /* SIG_TYPE_NOT_SET */ { SIG_PROP_FLOW_ACTION_PACKET, }, /* SIG_TYPE_IPONLY */ { SIG_PROP_FLOW_ACTION_FLOW, }, @@ -122,6 +123,7 @@ const struct SignatureProperties signature_properties[SIG_TYPE_MAX] = { /* SIG_TYPE_APPLAYER */ { SIG_PROP_FLOW_ACTION_FLOW, }, /* SIG_TYPE_APP_TX */ { SIG_PROP_FLOW_ACTION_FLOW, }, }; +// rule types documentation tag end: SignatureProperties // clang-format on /** \brief register inspect engine at start up time diff --git a/src/detect-http-cookie.c b/src/detect-http-cookie.c index 7397edf07955..865a88d8a1e2 100644 --- a/src/detect-http-cookie.c +++ b/src/detect-http-cookie.c @@ -181,7 +181,7 @@ static InspectionBuffer *GetRequestData(DetectEngineThreadCtx *det_ctx, if (htp_tx_request_headers(tx) == NULL) return NULL; - htp_header_t *h = (htp_header_t *)htp_tx_request_header(tx, "Cookie"); + const htp_header_t *h = htp_tx_request_header(tx, "Cookie"); if (h == NULL || htp_header_value(h) == NULL) { SCLogDebug("HTTP cookie header not present in this request"); return NULL; @@ -208,7 +208,7 @@ static InspectionBuffer *GetResponseData(DetectEngineThreadCtx *det_ctx, if (htp_tx_response_headers(tx) == NULL) return NULL; - htp_header_t *h = (htp_header_t *)htp_tx_response_header(tx, "Set-Cookie"); + const htp_header_t *h = htp_tx_response_header(tx, "Set-Cookie"); if (h == NULL || htp_header_value(h) == NULL) { SCLogDebug("HTTP cookie header not present in this request"); return NULL; diff --git a/src/detect-http-header-names.c b/src/detect-http-header-names.c index 84f3fe5e7e42..6f0d30219536 100644 --- a/src/detect-http-header-names.c +++ b/src/detect-http-header-names.c @@ -86,7 +86,7 @@ static uint8_t *GetBufferForTX( return NULL; } - htp_table_t *headers; + const htp_headers_t *headers; if (flags & STREAM_TOSERVER) { if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, flags) <= HTP_REQUEST_PROGRESS_HEADERS) @@ -103,9 +103,9 @@ static uint8_t *GetBufferForTX( /* fill the buffer. \r\nName1\r\nName2\r\n\r\n */ size_t i = 0; - size_t no_of_headers = htp_table_size(headers); + size_t no_of_headers = htp_headers_size(headers); for (; i < no_of_headers; i++) { - htp_header_t *h = htp_table_get_index(headers, i, NULL); + const htp_header_t *h = htp_headers_get_index(headers, i); size_t size = htp_header_name_len(h) + 2; // for \r\n if (i == 0) size += 2; diff --git a/src/detect-http-header.c b/src/detect-http-header.c index 81fde351be65..4c7d2fbc9593 100644 --- a/src/detect-http-header.c +++ b/src/detect-http-header.c @@ -79,7 +79,7 @@ static uint8_t *GetBufferForTX( return NULL; } - htp_table_t *headers; + const htp_headers_t *headers; if (flags & STREAM_TOSERVER) { if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, flags) <= HTP_REQUEST_PROGRESS_HEADERS) @@ -95,9 +95,9 @@ static uint8_t *GetBufferForTX( return NULL; size_t i = 0; - size_t no_of_headers = htp_table_size(headers); + size_t no_of_headers = htp_headers_size(headers); for (; i < no_of_headers; i++) { - htp_header_t *h = htp_table_get_index(headers, i, NULL); + const htp_header_t *h = htp_headers_get_index(headers, i); size_t size1 = htp_header_name_len(h); size_t size2 = htp_header_value_len(h); @@ -552,13 +552,13 @@ static InspectionBuffer *GetHttp1HeaderData(DetectEngineThreadCtx *det_ctx, } htp_tx_t *tx = (htp_tx_t *)txv; - htp_table_t *headers; + const htp_headers_t *headers; if (flags & STREAM_TOSERVER) { headers = htp_tx_request_headers(tx); } else { headers = htp_tx_response_headers(tx); } - size_t no_of_headers = htp_table_size(headers); + size_t no_of_headers = htp_headers_size(headers); if (local_id == 0) { // We initialize a big buffer on first item // Then, we will just use parts of it @@ -575,7 +575,7 @@ static InspectionBuffer *GetHttp1HeaderData(DetectEngineThreadCtx *det_ctx, hdr_td->cap = no_of_headers; } for (size_t i = 0; i < no_of_headers; i++) { - htp_header_t *h = htp_table_get_index(headers, i, NULL); + const htp_header_t *h = htp_headers_get_index(headers, i); size_t size1 = htp_header_name_len(h); size_t size2 = htp_header_value_len(h); size_t size = size1 + size2 + 2; diff --git a/src/detect-http-headers-stub.h b/src/detect-http-headers-stub.h index cadae73e0824..2dfcda843493 100644 --- a/src/detect-http-headers-stub.h +++ b/src/detect-http-headers-stub.h @@ -57,7 +57,7 @@ static InspectionBuffer *GetRequestData(DetectEngineThreadCtx *det_ctx, if (htp_tx_request_headers(tx) == NULL) return NULL; - htp_header_t *h = (htp_header_t *)htp_tx_request_header(tx, HEADER_NAME); + const htp_header_t *h = htp_tx_request_header(tx, HEADER_NAME); if (h == NULL || htp_header_value(h) == NULL) { SCLogDebug("HTTP %s header not present in this request", HEADER_NAME); @@ -112,7 +112,7 @@ static InspectionBuffer *GetResponseData(DetectEngineThreadCtx *det_ctx, if (htp_tx_response_headers(tx) == NULL) return NULL; - htp_header_t *h = (htp_header_t *)htp_tx_response_header(tx, HEADER_NAME); + const htp_header_t *h = htp_tx_response_header(tx, HEADER_NAME); if (h == NULL || htp_header_value(h) == NULL) { SCLogDebug("HTTP %s header not present in this request", HEADER_NAME); diff --git a/src/detect-http-host.c b/src/detect-http-host.c index 0df8911eeb49..a86625d8afae 100644 --- a/src/detect-http-host.c +++ b/src/detect-http-host.c @@ -352,7 +352,7 @@ static InspectionBuffer *GetRawData(DetectEngineThreadCtx *det_ctx, if (htp_tx_request_headers(tx) == NULL) return NULL; - htp_header_t *h = (htp_header_t *)htp_tx_request_header(tx, "Host"); + const htp_header_t *h = htp_tx_request_header(tx, "Host"); if (h == NULL || htp_header_value(h) == NULL) return NULL; diff --git a/src/detect-http-protocol.c b/src/detect-http-protocol.c index 8f50693be0cc..0c2d9cb71b62 100644 --- a/src/detect-http-protocol.c +++ b/src/detect-http-protocol.c @@ -87,7 +87,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, { InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { - bstr *str = NULL; + const bstr *str = NULL; htp_tx_t *tx = (htp_tx_t *)txv; if (flow_flags & STREAM_TOSERVER) diff --git a/src/detect-http-start.c b/src/detect-http-start.c index d1f7708a15c6..3f5920213d0e 100644 --- a/src/detect-http-start.c +++ b/src/detect-http-start.c @@ -85,8 +85,8 @@ static uint8_t *GetBufferForTX( return NULL; } - bstr *line = NULL; - htp_table_t *headers; + const bstr *line = NULL; + const htp_headers_t *headers; if (flags & STREAM_TOSERVER) { if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, flags) <= HTP_REQUEST_PROGRESS_HEADERS) @@ -115,9 +115,9 @@ static uint8_t *GetBufferForTX( buf->buffer[buf->len++] = '\n'; size_t i = 0; - size_t no_of_headers = htp_table_size(headers); + size_t no_of_headers = htp_headers_size(headers); for (; i < no_of_headers; i++) { - htp_header_t *h = htp_table_get_index(headers, i, NULL); + const htp_header_t *h = htp_headers_get_index(headers, i); size_t size1 = htp_header_name_len(h); size_t size2 = htp_header_value_len(h); size_t size = size1 + size2 + 4; diff --git a/src/detect-http-ua.c b/src/detect-http-ua.c index a643bf913fb5..2a539d69df3f 100644 --- a/src/detect-http-ua.c +++ b/src/detect-http-ua.c @@ -165,7 +165,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, if (htp_tx_request_headers(tx) == NULL) return NULL; - htp_header_t *h = (htp_header_t *)htp_tx_request_header(tx, "User-Agent"); + const htp_header_t *h = htp_tx_request_header(tx, "User-Agent"); if (h == NULL || htp_header_value(h) == NULL) { SCLogDebug("HTTP UA header not present in this request"); return NULL; diff --git a/src/detect.h b/src/detect.h index 804d3d7b1485..7408ec8edc4e 100644 --- a/src/detect.h +++ b/src/detect.h @@ -59,6 +59,7 @@ struct SCSigOrderFunc_; /* Forward declarations for structures from Rust. */ typedef struct SCDetectRequiresStatus SCDetectRequiresStatus; +// rule types documentation tag start: SignatureType enum SignatureType { SIG_TYPE_NOT_SET = 0, SIG_TYPE_IPONLY, // rule is handled by IPONLY engine @@ -76,6 +77,7 @@ enum SignatureType { SIG_TYPE_MAX, }; +// rule types documentation tag end: SignatureType enum SignaturePropertyFlowAction { SIG_PROP_FLOW_ACTION_PACKET, diff --git a/src/flow-hash.c b/src/flow-hash.c index c08b6e12c7ee..21a39b3cfdde 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -130,7 +130,10 @@ uint32_t FlowGetIpPairProtoHash(const Packet *p) fhk.ports[1] = 0xba98; fhk.proto = (uint8_t)p->proto; - fhk.recur = (uint8_t)p->recursion_level; + /* g_recurlvl_mask sets the recursion_level to 0 if + * decoder.recursion-level.use-for-tracking is disabled. + */ + fhk.recur = (uint8_t)p->recursion_level & g_recurlvl_mask; /* g_vlan_mask sets the vlan_ids to 0 if vlan.use-for-tracking * is disabled. */ fhk.vlan_id[0] = p->vlan_id[0] & g_vlan_mask; @@ -165,7 +168,7 @@ uint32_t FlowGetIpPairProtoHash(const Packet *p) fhk.ports[0] = 0xfedc; fhk.ports[1] = 0xba98; fhk.proto = (uint8_t)p->proto; - fhk.recur = (uint8_t)p->recursion_level; + fhk.recur = (uint8_t)p->recursion_level & g_recurlvl_mask; fhk.vlan_id[0] = p->vlan_id[0] & g_vlan_mask; fhk.vlan_id[1] = p->vlan_id[1] & g_vlan_mask; fhk.vlan_id[2] = p->vlan_id[2] & g_vlan_mask; @@ -205,7 +208,10 @@ static inline uint32_t FlowGetHash(const Packet *p) fhk.ports[pi] = p->dp; fhk.proto = p->proto; - fhk.recur = p->recursion_level; + /* g_recurlvl_mask sets the recursion_level to 0 if + * decoder.recursion-level.use-for-tracking is disabled. + */ + fhk.recur = p->recursion_level & g_recurlvl_mask; /* g_livedev_mask sets the livedev ids to 0 if livedev.use-for-tracking * is disabled. */ uint16_t devid = p->livedev ? p->livedev->id : 0; @@ -232,7 +238,7 @@ static inline uint32_t FlowGetHash(const Packet *p) fhk.ports[pi] = p->l4.vars.icmpv4.emb_dport; fhk.proto = ICMPV4_GET_EMB_PROTO(p); - fhk.recur = p->recursion_level; + fhk.recur = p->recursion_level & g_recurlvl_mask; uint16_t devid = p->livedev ? p->livedev->id : 0; fhk.livedev = devid & g_livedev_mask; fhk.vlan_id[0] = p->vlan_id[0] & g_vlan_mask; @@ -249,7 +255,7 @@ static inline uint32_t FlowGetHash(const Packet *p) fhk.ports[0] = 0xfeed; fhk.ports[1] = 0xbeef; fhk.proto = p->proto; - fhk.recur = p->recursion_level; + fhk.recur = p->recursion_level & g_recurlvl_mask; uint16_t devid = p->livedev ? p->livedev->id : 0; fhk.livedev = devid & g_livedev_mask; fhk.vlan_id[0] = p->vlan_id[0] & g_vlan_mask; @@ -284,7 +290,7 @@ static inline uint32_t FlowGetHash(const Packet *p) fhk.ports[1-pi] = p->sp; fhk.ports[pi] = p->dp; fhk.proto = p->proto; - fhk.recur = p->recursion_level; + fhk.recur = p->recursion_level & g_recurlvl_mask; uint16_t devid = p->livedev ? p->livedev->id : 0; fhk.livedev = devid & g_livedev_mask; fhk.vlan_id[0] = p->vlan_id[0] & g_vlan_mask; @@ -322,7 +328,7 @@ uint32_t FlowKeyGetHash(FlowKey *fk) fhk.ports[pi] = fk->dp; fhk.proto = fk->proto; - fhk.recur = fk->recursion_level; + fhk.recur = fk->recursion_level & g_recurlvl_mask; fhk.livedev = fk->livedev_id & g_livedev_mask; fhk.vlan_id[0] = fk->vlan_id[0] & g_vlan_mask; fhk.vlan_id[1] = fk->vlan_id[1] & g_vlan_mask; @@ -358,7 +364,7 @@ uint32_t FlowKeyGetHash(FlowKey *fk) fhk.ports[1-pi] = fk->sp; fhk.ports[pi] = fk->dp; fhk.proto = fk->proto; - fhk.recur = fk->recursion_level; + fhk.recur = fk->recursion_level & g_recurlvl_mask; fhk.livedev = fk->livedev_id & g_livedev_mask; fhk.vlan_id[0] = fk->vlan_id[0] & g_vlan_mask; fhk.vlan_id[1] = fk->vlan_id[1] & g_vlan_mask; @@ -412,7 +418,8 @@ static inline bool CmpFlowPacket(const Flow *f, const Packet *p) const uint32_t *p_src = p->src.address.address_un_data32; const uint32_t *p_dst = p->dst.address.address_un_data32; return CmpAddrsAndPorts(f_src, f_dst, f->sp, f->dp, p_src, p_dst, p->sp, p->dp) && - f->proto == p->proto && f->recursion_level == p->recursion_level && + f->proto == p->proto && + (f->recursion_level == p->recursion_level || g_recurlvl_mask == 0) && CmpVlanIds(f->vlan_id, p->vlan_id) && (f->livedev == p->livedev || g_livedev_mask == 0); } @@ -423,7 +430,8 @@ static inline bool CmpFlowKey(const Flow *f, const FlowKey *k) const uint32_t *k_src = k->src.address.address_un_data32; const uint32_t *k_dst = k->dst.address.address_un_data32; return CmpAddrsAndPorts(f_src, f_dst, f->sp, f->dp, k_src, k_dst, k->sp, k->dp) && - f->proto == k->proto && f->recursion_level == k->recursion_level && + f->proto == k->proto && + (f->recursion_level == k->recursion_level || g_recurlvl_mask == 0) && CmpVlanIds(f->vlan_id, k->vlan_id) && CmpLiveDevIds(f->livedev, k->livedev_id); } @@ -449,7 +457,8 @@ static inline bool CmpFlowICMPPacket(const Flow *f, const Packet *p) const uint32_t *p_dst = p->dst.address.address_un_data32; return CmpAddrsAndICMPTypes(f_src, f_dst, f->icmp_s.type, f->icmp_d.type, p_src, p_dst, p->icmp_s.type, p->icmp_d.type) && - f->proto == p->proto && f->recursion_level == p->recursion_level && + f->proto == p->proto && + (f->recursion_level == p->recursion_level || g_recurlvl_mask == 0) && CmpVlanIds(f->vlan_id, p->vlan_id) && (f->livedev == p->livedev || g_livedev_mask == 0); } @@ -472,7 +481,8 @@ static inline int FlowCompareICMPv4(Flow *f, const Packet *p) if ((f->src.addr_data32[0] == IPV4_GET_RAW_IPSRC_U32(PacketGetICMPv4EmbIPv4(p))) && (f->dst.addr_data32[0] == IPV4_GET_RAW_IPDST_U32(PacketGetICMPv4EmbIPv4(p))) && f->sp == p->l4.vars.icmpv4.emb_sport && f->dp == p->l4.vars.icmpv4.emb_dport && - f->proto == ICMPV4_GET_EMB_PROTO(p) && f->recursion_level == p->recursion_level && + f->proto == ICMPV4_GET_EMB_PROTO(p) && + (f->recursion_level == p->recursion_level || g_recurlvl_mask == 0) && CmpVlanIds(f->vlan_id, p->vlan_id) && (f->livedev == p->livedev || g_livedev_mask == 0)) { return 1; @@ -483,7 +493,8 @@ static inline int FlowCompareICMPv4(Flow *f, const Packet *p) (f->src.addr_data32[0] == IPV4_GET_RAW_IPDST_U32(PacketGetICMPv4EmbIPv4(p))) && f->dp == p->l4.vars.icmpv4.emb_sport && f->sp == p->l4.vars.icmpv4.emb_dport && f->proto == ICMPV4_GET_EMB_PROTO(p) && - f->recursion_level == p->recursion_level && CmpVlanIds(f->vlan_id, p->vlan_id) && + (f->recursion_level == p->recursion_level || g_recurlvl_mask == 0) && + CmpVlanIds(f->vlan_id, p->vlan_id) && (f->livedev == p->livedev || g_livedev_mask == 0)) { return 1; } @@ -514,8 +525,8 @@ static inline int FlowCompareESP(Flow *f, const Packet *p) const uint32_t *p_dst = p->dst.address.address_un_data32; return CmpAddrs(f_src, p_src) && CmpAddrs(f_dst, p_dst) && f->proto == p->proto && - f->recursion_level == p->recursion_level && CmpVlanIds(f->vlan_id, p->vlan_id) && - f->esp.spi == ESP_GET_SPI(PacketGetESP(p)) && + (f->recursion_level == p->recursion_level || g_recurlvl_mask == 0) && + CmpVlanIds(f->vlan_id, p->vlan_id) && f->esp.spi == ESP_GET_SPI(PacketGetESP(p)) && (f->livedev == p->livedev || g_livedev_mask == 0); } diff --git a/src/output-json-http.c b/src/output-json-http.c index c8e3723aec53..0fa6414668e2 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -220,39 +220,40 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx) if (htp_tx_request_headers(tx) != NULL) { /* user agent */ - htp_header_t *h_user_agent = htp_tx_request_header(tx, "user-agent"); + const htp_header_t *h_user_agent = htp_tx_request_header(tx, "user-agent"); if (h_user_agent != NULL) { - jb_set_string_from_bytes(js, "http_user_agent", bstr_ptr(h_user_agent->value), - (uint32_t)bstr_len(h_user_agent->value)); + jb_set_string_from_bytes(js, "http_user_agent", htp_header_value_ptr(h_user_agent), + (uint32_t)htp_header_value_len(h_user_agent)); } /* x-forwarded-for */ - htp_header_t *h_x_forwarded_for = htp_tx_request_header(tx, "x-forwarded-for"); + const htp_header_t *h_x_forwarded_for = htp_tx_request_header(tx, "x-forwarded-for"); if (h_x_forwarded_for != NULL) { - jb_set_string_from_bytes(js, "xff", bstr_ptr(h_x_forwarded_for->value), - (uint32_t)bstr_len(h_x_forwarded_for->value)); + jb_set_string_from_bytes(js, "xff", htp_header_value_ptr(h_x_forwarded_for), + (uint32_t)htp_header_value_len(h_x_forwarded_for)); } } /* content-type */ if (htp_tx_response_headers(tx) != NULL) { - htp_header_t *h_content_type = htp_tx_response_header(tx, "content-type"); + const htp_header_t *h_content_type = htp_tx_response_header(tx, "content-type"); if (h_content_type != NULL) { - const size_t size = bstr_len(h_content_type->value) * 2 + 1; + const size_t size = htp_header_value_len(h_content_type) * 2 + 1; char string[size]; - BytesToStringBuffer(bstr_ptr(h_content_type->value), bstr_len(h_content_type->value), string, size); + BytesToStringBuffer(htp_header_value_ptr(h_content_type), + htp_header_value_len(h_content_type), string, size); char *p = strchr(string, ';'); if (p != NULL) *p = '\0'; jb_set_string(js, "http_content_type", string); } - htp_header_t *h_content_range = htp_tx_response_header(tx, "content-range"); + const htp_header_t *h_content_range = htp_tx_response_header(tx, "content-range"); if (h_content_range != NULL) { jb_open_object(js, "content_range"); - jb_set_string_from_bytes(js, "raw", bstr_ptr(h_content_range->value), - (uint32_t)bstr_len(h_content_range->value)); + jb_set_string_from_bytes(js, "raw", htp_header_value_ptr(h_content_range), + (uint32_t)htp_header_value_len(h_content_range)); HTTPContentRange crparsed; - if (HTPParseContentRange(h_content_range->value, &crparsed) == 0) { + if (HTPParseContentRange(htp_header_value(h_content_range), &crparsed) == 0) { if (crparsed.start >= 0) jb_set_uint(js, "start", crparsed.start); if (crparsed.end >= 0) @@ -268,13 +269,13 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx) static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx) { /* referer */ - htp_header_t *h_referer = NULL; + const htp_header_t *h_referer = NULL; if (htp_tx_request_headers(tx) != NULL) { h_referer = htp_tx_request_header(tx, "referer"); } if (h_referer != NULL) { - jb_set_string_from_bytes( - js, "http_refer", bstr_ptr(h_referer->value), (uint32_t)bstr_len(h_referer->value)); + jb_set_string_from_bytes(js, "http_refer", htp_header_value_ptr(h_referer), + (uint32_t)htp_header_value_len(h_referer)); } /* method */ @@ -300,10 +301,10 @@ static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx) (uint32_t)bstr_len(htp_tx_response_status(tx))); } - htp_header_t *h_location = htp_tx_response_header(tx, "location"); + const htp_header_t *h_location = htp_tx_response_header(tx, "location"); if (h_location != NULL) { - jb_set_string_from_bytes( - js, "redirect", bstr_ptr(h_location->value), (uint32_t)bstr_len(h_location->value)); + jb_set_string_from_bytes(js, "redirect", htp_header_value_ptr(h_location), + (uint32_t)htp_header_value_len(h_location)); } /* length */ @@ -313,17 +314,17 @@ static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx) static void EveHttpLogJSONHeaders( JsonBuilder *js, uint32_t direction, htp_tx_t *tx, LogHttpFileCtx *http_ctx) { - htp_table_t *headers = direction & LOG_HTTP_REQ_HEADERS ? htp_tx_request_headers(tx) - : htp_tx_response_headers(tx); + const htp_headers_t *headers = direction & LOG_HTTP_REQ_HEADERS ? htp_tx_request_headers(tx) + : htp_tx_response_headers(tx); char name[MAX_SIZE_HEADER_NAME] = {0}; char value[MAX_SIZE_HEADER_VALUE] = {0}; - size_t n = htp_table_size(headers); + size_t n = htp_headers_size(headers); JsonBuilderMark mark = { 0, 0, 0 }; jb_get_mark(js, &mark); bool array_empty = true; jb_open_array(js, direction & LOG_HTTP_REQ_HEADERS ? "request_headers" : "response_headers"); for (size_t i = 0; i < n; i++) { - htp_header_t *h = htp_table_get_index(headers, i, NULL); + const htp_header_t *h = htp_headers_get_index(headers, i); if ((http_ctx->flags & direction) == 0 && http_ctx->fields != 0) { bool tolog = false; for (HttpField f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) { diff --git a/src/suricata.c b/src/suricata.c index 7c238c48cc5e..60b67f717b5e 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -202,6 +202,10 @@ uint16_t g_vlan_mask = 0xffff; * comparing flows */ uint16_t g_livedev_mask = 0xffff; +/** determine (without branching) if we include the recursion levels when hashing or + * comparing flows */ +uint8_t g_recurlvl_mask = 0xff; + /* flag to disable hashing almost globally, to be similar to disabling nss * support */ bool g_disable_hashing = false; @@ -2914,6 +2918,10 @@ void SuricataInit(void) /* Ignore livedev id when comparing flows. */ g_livedev_mask = 0x0000; } + if (ConfGetBool("decoder.recursion-level.use-for-tracking", &tracking) == 1 && !tracking) { + /* Ignore recursion level when comparing flows. */ + g_recurlvl_mask = 0x00; + } SetupUserMode(&suricata); InitRunAs(&suricata); diff --git a/src/suricata.h b/src/suricata.h index 70393541f6fc..70209f393e9a 100644 --- a/src/suricata.h +++ b/src/suricata.h @@ -173,6 +173,7 @@ extern volatile uint8_t suricata_ctl_flags; extern int g_disable_randomness; extern uint16_t g_vlan_mask; extern uint16_t g_livedev_mask; +extern uint8_t g_recurlvl_mask; /* Flag to disable hashing (almost) globally. */ extern bool g_disable_hashing; diff --git a/src/util-lua-http.c b/src/util-lua-http.c index a8924c42fb02..d4d056ad0342 100644 --- a/src/util-lua-http.c +++ b/src/util-lua-http.c @@ -154,7 +154,7 @@ static int HttpGetHeader(lua_State *luastate, int dir) if (name == NULL) return LuaCallbackError(luastate, "1st argument missing, empty or wrong type"); - htp_table_t *headers = htp_tx_request_headers(tx); + const htp_headers_t *headers = htp_tx_request_headers(tx); if (dir == 1) headers = htp_tx_response_headers(tx); if (headers == NULL) @@ -223,7 +223,7 @@ static int HttpGetHeaders(lua_State *luastate, int dir) if (tx == NULL) return LuaCallbackError(luastate, "internal error: no tx"); - htp_table_t *table = htp_tx_request_headers(tx); + const htp_headers_t *table = htp_tx_request_headers(tx); if (dir == 1) table = htp_tx_response_headers(tx); if (htp_tx_request_headers(tx) == NULL) @@ -232,9 +232,9 @@ static int HttpGetHeaders(lua_State *luastate, int dir) lua_newtable(luastate); htp_header_t *h = NULL; size_t i = 0; - size_t no_of_headers = htp_table_size(table); + size_t no_of_headers = htp_headers_size(table); for (; i < no_of_headers; i++) { - h = htp_table_get_index(table, i, NULL); + h = htp_headers_get_index(table, i); LuaPushStringBuffer(luastate, htp_header_name_ptr(h), htp_header_name_len(h)); LuaPushStringBuffer(luastate, htp_header_value_ptr(h), htp_header_value_len(h)); lua_settable(luastate, -3); diff --git a/src/util-print.c b/src/util-print.c index 16b66679f680..77d88faa64dd 100644 --- a/src/util-print.c +++ b/src/util-print.c @@ -66,7 +66,7 @@ void PrintRawLineHexBuf(char *retbuf, uint32_t retbuflen, const uint8_t *buf, ui } } -void PrintRawUriFp(FILE *fp, uint8_t *buf, uint32_t buflen) +void PrintRawUriFp(FILE *fp, const uint8_t *buf, uint32_t buflen) { #define BUFFER_LENGTH 2048 char nbuf[BUFFER_LENGTH] = ""; @@ -90,7 +90,8 @@ void PrintRawUriFp(FILE *fp, uint8_t *buf, uint32_t buflen) fprintf(fp, "%s", nbuf); } -void PrintRawUriBuf(char *retbuf, uint32_t *offset, uint32_t retbuflen, uint8_t *buf, size_t buflen) +void PrintRawUriBuf( + char *retbuf, uint32_t *offset, uint32_t retbuflen, const uint8_t *buf, size_t buflen) { for (size_t u = 0; u < buflen; u++) { if (isprint(buf[u]) && buf[u] != '\"') { diff --git a/src/util-print.h b/src/util-print.h index c9f19b4cdb2d..e589a0a2ddba 100644 --- a/src/util-print.h +++ b/src/util-print.h @@ -40,8 +40,8 @@ } while (0) void PrintBufferRawLineHex(char *, int *,int, const uint8_t *, uint32_t); -void PrintRawUriFp(FILE *, uint8_t *, uint32_t); -void PrintRawUriBuf(char *, uint32_t *, uint32_t, uint8_t *, size_t); +void PrintRawUriFp(FILE *, const uint8_t *, uint32_t); +void PrintRawUriBuf(char *, uint32_t *, uint32_t, const uint8_t *, size_t); void PrintRawDataFp(FILE *, const uint8_t *, uint32_t); void PrintRawDataToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, uint32_t src_buf_len); diff --git a/suricata.yaml.in b/suricata.yaml.in index 4bc9e87aa2af..46c94f0defbb 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1673,6 +1673,13 @@ decoder: # maximum number of decoder layers for a packet # max-layers: 16 + # This option controls the use of packet recursion level in the flow + # (and defrag) hashing. This is enabled by default and should be + # disabled if packet pickup of tunneled packets occurs before the kernel + # has put the headers on, like when using netmap driver pickup. + recursion-level: + use-for-tracking: true + ## ## Performance tuning and profiling ##