Add pipeline for LEEF syslog format

criblpacks · Feb 4, 2022 · 15fe2df · 15fe2df
1 parent a9ddf09
commit 15fe2df
Show file tree

Hide file tree

Showing 5 changed files with 165 additions and 1 deletion.
diff --git a/data/samples/zy4MUq.json b/data/samples/zy4MUq.json
@@ -0,0 +1 @@
+[{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:2.0|ExtraHop|Reveal(x)|7.8|extrahop-detection|xa6|appliance_id=dd0685a93cdc3bc5bb9a88251e7a7804¦categories=sec,sec.caution¦det_id=0123456789¦det_url=https://cribl.cloud.extrahop.com/extrahop/#/detections/detail/0123456789¦update_time=Jan 24 2022 20:23:39 +0000¦end_time=Jan 24 2022 20:23:39 +0000¦risk_score=60¦start_time=Jan 24 2022 20:23:39 +0000¦title=Non-standard HTTP Port¦offender_ip=10.1.2.3¦victim_ip=192.168.1.50¦offender_id=dd0685a93cdc3bc5¦desc=Host CRIBL sent an HTTP request to an external device on a non-standard HTTP port.","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:2.0|ExtraHop|Reveal(x)|7.8|extrahop-detection|xa6|appliance_id=dd0685a93cdc3bc5bb9a88251e7a7804¦categories=sec,sec.caution¦det_id=0123456789¦det_url=https://cribl.cloud.extrahop.com/extrahop/#/detections/detail/0123456789¦update_time=Jan 24 2022 20:23:39 +0000¦end_time=Jan 24 2022 20:23:39 +0000¦risk_score=60¦start_time=Jan 24 2022 20:23:39 +0000¦title=Non-standard HTTP Port¦offender_ip=10.1.2.3¦victim_ip=192.168.1.50¦offender_id=dd0685a93cdc3bc5¦desc=Host CRIBL sent an HTTP request to an external device on a non-standard HTTP port."},{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:1.0|NXLog|in|3.0.1775|unknown|EventReceivedTime=2016-09-13 11:23:12\tSourceModuleName=in\tSourceModuleType=im_file\tdevTime=2016-09-13 11:23:11\tidentHostName=myserver\tPurpose=test\tMessage=This is a test log message.\tdevTimeFormat=yyyy-MM-dd HH:mm:ss","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:1.0|NXLog|in|3.0.1775|unknown|EventReceivedTime=2016-09-13 11:23:12\tSourceModuleName=in\tSourceModuleType=im_file\tdevTime=2016-09-13 11:23:11\tidentHostName=myserver\tPurpose=test\tMessage=This is a test log message.\tdevTimeFormat=yyyy-MM-dd HH:mm:ss"},{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:1.0|CB|CB|5.1.0.150625.500|watchlist.hit.process|cb_server=None\tchildproc_count=0\tcmdline=\"C:\\\\Program Files\\\\Bit9\\\\Parity Agent\\\\Parity.exe\" /service /server bit9server.bit9se.com /port 41002\tcomms_ip=-1062672891\tcrossproc_count=0\tfilemod_count=5\tgroup=Default Group\thost_type=workstation\thostname=W7-LOW\tid=00000016-0000-0608-01d1-171453a78ab2\tinterface_ip=192.168.230.5\tlast_update=2015-11-04T15:20:56.939Z\tmodload_count=371\tnetconn_count=1\tos_type=windows\tparent_guid=00000016-0000-0210-01d1-1714513a9f24\tparent_md5=000000000000000000000000000000\tparent_name=services.exe\tparent_pid=528\tparent_unique_id=00000016-0000-0210-01d1-1714513a9f24-00000001\tpath=c:\\\\program files\\\\bit9\\\\parity agent\\\\parity.exe\tprocess_guid=00000016-0000-0608-01d1-171453a78ab2\tprocess_md5=15785fcb9495aa518c8c751e80ab9bf7\tprocess_name=parity.exe\tprocess_pid=1544\tregmod_count=8\tsegment_id=1\tsensor_id=22\tserver_name=cbserver\tstart=2015-11-04T15:20:24.072Z\ttimestamp=1446092826.89\ttype=watchlist.hit.process\tunique_id=00000016-0000-0608-01d1-171453a78ab2-00000001\tusername=SYSTEM\twatchlist_id=7\twatchlist_name=Non-System Filemods to system32","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:1.0|CB|CB|5.1.0.150625.500|watchlist.hit.process|cb_server=None\tchildproc_count=0\tcmdline=\"C:\\\\Program Files\\\\Bit9\\\\Parity Agent\\\\Parity.exe\" /service /server bit9server.bit9se.com /port 41002\tcomms_ip=-1062672891\tcrossproc_count=0\tfilemod_count=5\tgroup=Default Group\thost_type=workstation\thostname=W7-LOW\tid=00000016-0000-0608-01d1-171453a78ab2\tinterface_ip=192.168.230.5\tlast_update=2015-11-04T15:20:56.939Z\tmodload_count=371\tnetconn_count=1\tos_type=windows\tparent_guid=00000016-0000-0210-01d1-1714513a9f24\tparent_md5=000000000000000000000000000000\tparent_name=services.exe\tparent_pid=528\tparent_unique_id=00000016-0000-0210-01d1-1714513a9f24-00000001\tpath=c:\\\\program files\\\\bit9\\\\parity agent\\\\parity.exe\tprocess_guid=00000016-0000-0608-01d1-171453a78ab2\tprocess_md5=15785fcb9495aa518c8c751e80ab9bf7\tprocess_name=parity.exe\tprocess_pid=1544\tregmod_count=8\tsegment_id=1\tsensor_id=22\tserver_name=cbserver\tstart=2015-11-04T15:20:24.072Z\ttimestamp=1446092826.89\ttype=watchlist.hit.process\tunique_id=00000016-0000-0608-01d1-171453a78ab2-00000001\tusername=SYSTEM\twatchlist_id=7\twatchlist_name=Non-System Filemods to system32"},{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:1.0|Aruba Networks|ClearPass|6.5.0.68878|1309854-1-0|RADIUS.Acct-Calling-Station-Id=00:88:57:2d:12:a4\tRADIUS.Acct-Framed-IP-Address=192.167.203.170\tRADIUS.Auth-Source=AD:10.17.4.130\tRADIUS.Acct-Timestamp=2017-12-02 15:32:47+05:30\tRADIUS.Auth-Method=PAP\tRADIUS.Acct-Service-Name=Authenticate-Only\tRADIUS.Acct-Session-Time=565\tTimestampFormat=MMM dd yyyy HH:mm:ss.SSS z\tRADIUS.Acct-NAS-Port=0\tRADIUS.Acct-Session-Id=R000a5038-01-547d8e47\tRADIUS.Acct-NAS-Port-Type=Wireless-802.11\tRADIUS.Acct-Output-Octets=412895267\tRADIUS.Acct-Username=A_user706\tRADIUS.Acct-NAS-IP-Address=10.17.6.124\tRADIUS.Acct-Input-Octets=665942581","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:1.0|Aruba Networks|ClearPass|6.5.0.68878|1309854-1-0|RADIUS.Acct-Calling-Station-Id=00:88:57:2d:12:a4\tRADIUS.Acct-Framed-IP-Address=192.167.203.170\tRADIUS.Auth-Source=AD:10.17.4.130\tRADIUS.Acct-Timestamp=2017-12-02 15:32:47+05:30\tRADIUS.Auth-Method=PAP\tRADIUS.Acct-Service-Name=Authenticate-Only\tRADIUS.Acct-Session-Time=565\tTimestampFormat=MMM dd yyyy HH:mm:ss.SSS z\tRADIUS.Acct-NAS-Port=0\tRADIUS.Acct-Session-Id=R000a5038-01-547d8e47\tRADIUS.Acct-NAS-Port-Type=Wireless-802.11\tRADIUS.Acct-Output-Octets=412895267\tRADIUS.Acct-Username=A_user706\tRADIUS.Acct-NAS-IP-Address=10.17.6.124\tRADIUS.Acct-Input-Octets=665942581"}]
diff --git a/default/pipelines/cef_processor/conf.yml b/default/pipelines/cef_processor/conf.yml
@@ -1,4 +1,5 @@
 output: default
+description: Common Event Format (CEF) Reshaping Pipeline
 groups:
   BVYPDJ:
     name: Field Extractions

diff --git a/default/pipelines/leef_processor/conf.yml b/default/pipelines/leef_processor/conf.yml
@@ -0,0 +1,141 @@
+output: default
+description: Log Event Extended Format (LEEF) Reshaping Pipeline
+groups:
+  Nxi2NQ:
+    name: LEEF Data Extraction
+    description: Extract headers and message body from the Syslog message and create event
+      fields
+    index: 1
+  FPIzG6:
+    name: Re-serialize as K=V Pairs (No LEEF header)
+    disabled: true
+    index: 2
+  Og9wZN:
+    name: Re-serialize as JSON
+    disabled: true
+    index: 3
+asyncFuncTimeout: 1000
+functions:
+  - id: comment
+    filter: "true"
+    disabled: null
+    conf:
+      comment: |-
+        Log Event Extended Format (LEEF) handling
+
+        Author: Brendan Dalpe ([email protected])
+  - id: regex_extract
+    filter: "true"
+    disabled: false
+    conf:
+      source: _raw
+      iterations: 100
+      overwrite: false
+      regex: /(?<__header>LEEF:(?<leef_version>[\d\.]+)\|(?<leef_device_vendor>.*?)(?<!\\)\|(?<leef_device_product>.*?)(?<!\\)\|(?<leef_device_version>.*?)(?<!\\)\|(?<leef_event_id>.*?)(?<!\\)\|(?:(?<leef_delimiter_character>(?:\S|x[0-9a-fA-F]+))(?<!\\)\|)?)(?<__raw_message>.*)/
+    description: Extract LEEF header values
+    groupId: Nxi2NQ
+  - id: mask
+    filter: leef_delimiter_character
+    disabled: null
+    conf:
+      rules:
+        - matchRegex: /(.+)/
+          replaceExpr: String.fromCharCode(C.Decode.hex(`0${g1}`))
+      fields:
+        - leef_delimiter_character
+    description: Convert LEEF delimiter character from hex to ASCII
+    groupId: Nxi2NQ
+  - id: comment
+    filter: "true"
+    disabled: null
+    conf:
+      comment: K=V field extraction happens here 👇
+    groupId: Nxi2NQ
+  - id: code
+    filter: "true"
+    disabled: null
+    conf:
+      maxNumOfIterations: 5000
+      code: >-
+        // Split raw message based on the delimiter character
+
+        let arr = __e['__raw_message'].split(__e['leef_delimiter_character'] || '\t')
+
+
+        arr.forEach(entry => {
+          // Extract all key value pairs
+          let e = entry.match(/([^=]+?)=(.+)/)
+
+          // Map key=value pairs and place into the event data
+          __e[e[1]] = e[2]
+        })
+    description: Split and Extract K=V Pairs from LEEF message
+    groupId: Nxi2NQ
+  - id: serialize
+    filter: "true"
+    disabled: true
+    conf:
+      type: kvp
+      fields:
+        - "!__*"
+        - "!_*"
+        - "!cribl*"
+        - "!leef_*"
+        - "!index"
+        - "!host"
+        - "!source"
+        - "!sourcetype"
+        - "!message"
+        - "!severity*"
+        - "!facility*"
+        - "!appname"
+        - "*"
+      dstField: _raw
+      cleanFields: true
+    groupId: FPIzG6
+  - id: serialize
+    filter: "true"
+    disabled: true
+    conf:
+      type: json
+      dstField: _raw
+      fields:
+        - "!__*"
+        - "!_*"
+        - "!cribl*"
+        - "!leef_*"
+        - "!index"
+        - "!host"
+        - "!source"
+        - "!sourcetype"
+        - "!message"
+        - "!severity*"
+        - "!facility*"
+        - "!appname"
+        - "*"
+      cleanFields: true
+    groupId: Og9wZN
+  - id: eval
+    filter: "true"
+    disabled: false
+    conf:
+      add:
+        - name: index
+          value: "index ? index : 'main'"
+        - name: source
+          value: "source ? source : 'cribl'"
+        - name: sourcetype
+          value: "sourcetype ? sourcetype : 'leef:syslog'"
+        - name: host
+          value: "host ? host : 'cribl'"
+      keep:
+        - _*
+        - cribl*
+        - index
+        - source
+        - sourcetype
+        - host
+      remove:
+        - "*"
+        - leef_*
+    description: Cleanup fields and set Splunk metadata fields
diff --git a/default/pipelines/route.yml b/default/pipelines/route.yml
@@ -8,7 +8,20 @@ routes:
     pipeline: cef_processor
     description: ""
     clones: []
-    filter: _raw.includes('CEF:')
+    enableOutputExpression: false
+    outputExpression: null
+    filter: _raw.indexOf(' CEF:') > -1
+    output: default
+  - id: LrkMwA
+    name: Process LEEF events
+    final: true
+    disabled: false
+    pipeline: leef_processor
+    description: ""
+    clones: []
+    enableOutputExpression: false
+    outputExpression: null
+    filter: _raw.indexOf(' LEEF:') > -1
     output: default
   - id: idTnkh
     name: Blackhole non-CEF events
@@ -17,5 +30,7 @@ routes:
     pipeline: devnull
     description: ""
     clones: []
+    enableOutputExpression: false
+    outputExpression: null
     filter: "true"
     output: default
diff --git a/default/samples.yml b/default/samples.yml
@@ -3,3 +3,9 @@ Uvsiga:
   created: 1631995176427
   size: 12864
   numEvents: 12
+zy4MUq:
+  sampleName: leef_syslog_samples.log
+  created: 1644009826624
+  size: 6163
+  numEvents: 4
+  modified: 1644009826620.8125
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		[{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:2.0\|ExtraHop\|Reveal(x)\|7.8\|extrahop-detection\|xa6\|appliance_id=dd0685a93cdc3bc5bb9a88251e7a7804¦categories=sec,sec.caution¦det_id=0123456789¦det_url=https://cribl.cloud.extrahop.com/extrahop/#/detections/detail/0123456789¦update_time=Jan 24 2022 20:23:39 +0000¦end_time=Jan 24 2022 20:23:39 +0000¦risk_score=60¦start_time=Jan 24 2022 20:23:39 +0000¦title=Non-standard HTTP Port¦offender_ip=10.1.2.3¦victim_ip=192.168.1.50¦offender_id=dd0685a93cdc3bc5¦desc=Host CRIBL sent an HTTP request to an external device on a non-standard HTTP port.","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:2.0\|ExtraHop\|Reveal(x)\|7.8\|extrahop-detection\|xa6\|appliance_id=dd0685a93cdc3bc5bb9a88251e7a7804¦categories=sec,sec.caution¦det_id=0123456789¦det_url=https://cribl.cloud.extrahop.com/extrahop/#/detections/detail/0123456789¦update_time=Jan 24 2022 20:23:39 +0000¦end_time=Jan 24 2022 20:23:39 +0000¦risk_score=60¦start_time=Jan 24 2022 20:23:39 +0000¦title=Non-standard HTTP Port¦offender_ip=10.1.2.3¦victim_ip=192.168.1.50¦offender_id=dd0685a93cdc3bc5¦desc=Host CRIBL sent an HTTP request to an external device on a non-standard HTTP port."},{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:1.0\|NXLog\|in\|3.0.1775\|unknown\|EventReceivedTime=2016-09-13 11:23:12\tSourceModuleName=in\tSourceModuleType=im_file\tdevTime=2016-09-13 11:23:11\tidentHostName=myserver\tPurpose=test\tMessage=This is a test log message.\tdevTimeFormat=yyyy-MM-dd HH:mm:ss","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:1.0\|NXLog\|in\|3.0.1775\|unknown\|EventReceivedTime=2016-09-13 11:23:12\tSourceModuleName=in\tSourceModuleType=im_file\tdevTime=2016-09-13 11:23:11\tidentHostName=myserver\tPurpose=test\tMessage=This is a test log message.\tdevTimeFormat=yyyy-MM-dd HH:mm:ss"},{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:1.0\|CB\|CB\|5.1.0.150625.500\|watchlist.hit.process\|cb_server=None\tchildproc_count=0\tcmdline=\"C:\\\\Program Files\\\\Bit9\\\\Parity Agent\\\\Parity.exe\" /service /server bit9server.bit9se.com /port 41002\tcomms_ip=-1062672891\tcrossproc_count=0\tfilemod_count=5\tgroup=Default Group\thost_type=workstation\thostname=W7-LOW\tid=00000016-0000-0608-01d1-171453a78ab2\tinterface_ip=192.168.230.5\tlast_update=2015-11-04T15:20:56.939Z\tmodload_count=371\tnetconn_count=1\tos_type=windows\tparent_guid=00000016-0000-0210-01d1-1714513a9f24\tparent_md5=000000000000000000000000000000\tparent_name=services.exe\tparent_pid=528\tparent_unique_id=00000016-0000-0210-01d1-1714513a9f24-00000001\tpath=c:\\\\program files\\\\bit9\\\\parity agent\\\\parity.exe\tprocess_guid=00000016-0000-0608-01d1-171453a78ab2\tprocess_md5=15785fcb9495aa518c8c751e80ab9bf7\tprocess_name=parity.exe\tprocess_pid=1544\tregmod_count=8\tsegment_id=1\tsensor_id=22\tserver_name=cbserver\tstart=2015-11-04T15:20:24.072Z\ttimestamp=1446092826.89\ttype=watchlist.hit.process\tunique_id=00000016-0000-0608-01d1-171453a78ab2-00000001\tusername=SYSTEM\twatchlist_id=7\twatchlist_name=Non-System Filemods to system32","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:1.0\|CB\|CB\|5.1.0.150625.500\|watchlist.hit.process\|cb_server=None\tchildproc_count=0\tcmdline=\"C:\\\\Program Files\\\\Bit9\\\\Parity Agent\\\\Parity.exe\" /service /server bit9server.bit9se.com /port 41002\tcomms_ip=-1062672891\tcrossproc_count=0\tfilemod_count=5\tgroup=Default Group\thost_type=workstation\thostname=W7-LOW\tid=00000016-0000-0608-01d1-171453a78ab2\tinterface_ip=192.168.230.5\tlast_update=2015-11-04T15:20:56.939Z\tmodload_count=371\tnetconn_count=1\tos_type=windows\tparent_guid=00000016-0000-0210-01d1-1714513a9f24\tparent_md5=000000000000000000000000000000\tparent_name=services.exe\tparent_pid=528\tparent_unique_id=00000016-0000-0210-01d1-1714513a9f24-00000001\tpath=c:\\\\program files\\\\bit9\\\\parity agent\\\\parity.exe\tprocess_guid=00000016-0000-0608-01d1-171453a78ab2\tprocess_md5=15785fcb9495aa518c8c751e80ab9bf7\tprocess_name=parity.exe\tprocess_pid=1544\tregmod_count=8\tsegment_id=1\tsensor_id=22\tserver_name=cbserver\tstart=2015-11-04T15:20:24.072Z\ttimestamp=1446092826.89\ttype=watchlist.hit.process\tunique_id=00000016-0000-0608-01d1-171453a78ab2-00000001\tusername=SYSTEM\twatchlist_id=7\twatchlist_name=Non-System Filemods to system32"},{"_time":1643734800,"cribl_breaker":"Break on newlines","message":"LEEF:1.0\|Aruba Networks\|ClearPass\|6.5.0.68878\|1309854-1-0\|RADIUS.Acct-Calling-Station-Id=00:88:57:2d:12:a4\tRADIUS.Acct-Framed-IP-Address=192.167.203.170\tRADIUS.Auth-Source=AD:10.17.4.130\tRADIUS.Acct-Timestamp=2017-12-02 15:32:47+05:30\tRADIUS.Auth-Method=PAP\tRADIUS.Acct-Service-Name=Authenticate-Only\tRADIUS.Acct-Session-Time=565\tTimestampFormat=MMM dd yyyy HH:mm:ss.SSS z\tRADIUS.Acct-NAS-Port=0\tRADIUS.Acct-Session-Id=R000a5038-01-547d8e47\tRADIUS.Acct-NAS-Port-Type=Wireless-802.11\tRADIUS.Acct-Output-Octets=412895267\tRADIUS.Acct-Username=A_user706\tRADIUS.Acct-NAS-IP-Address=10.17.6.124\tRADIUS.Acct-Input-Octets=665942581","severity":6,"facility":16,"host":"logstream","appname":"syslog","severityName":"info","facilityName":"local0","_raw":"<134>Feb 01 05:00:00 logstream syslog: LEEF:1.0\|Aruba Networks\|ClearPass\|6.5.0.68878\|1309854-1-0\|RADIUS.Acct-Calling-Station-Id=00:88:57:2d:12:a4\tRADIUS.Acct-Framed-IP-Address=192.167.203.170\tRADIUS.Auth-Source=AD:10.17.4.130\tRADIUS.Acct-Timestamp=2017-12-02 15:32:47+05:30\tRADIUS.Auth-Method=PAP\tRADIUS.Acct-Service-Name=Authenticate-Only\tRADIUS.Acct-Session-Time=565\tTimestampFormat=MMM dd yyyy HH:mm:ss.SSS z\tRADIUS.Acct-NAS-Port=0\tRADIUS.Acct-Session-Id=R000a5038-01-547d8e47\tRADIUS.Acct-NAS-Port-Type=Wireless-802.11\tRADIUS.Acct-Output-Octets=412895267\tRADIUS.Acct-Username=A_user706\tRADIUS.Acct-NAS-IP-Address=10.17.6.124\tRADIUS.Acct-Input-Octets=665942581"}]