Helm multiline passer template and usage

apis/fluentbit/v1alpha2/plugins/filter/lua_types.go

@@ -1,9 +1,9 @@
 package filter
 
 import (
+	"regexp"
 	"strconv"
 	"strings"
-	"regexp"
 
 	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
 	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
@@ -17,12 +17,12 @@ import (
 type Lua struct {
 	plugins.CommonParams `json:",inline"`
 	// Path to the Lua script that will be used.
-	Script v1.ConfigMapKeySelector `json:"script"`
+	Script v1.ConfigMapKeySelector `json:"script,omitempty"`
 	// Lua function name that will be triggered to do filtering.
 	// It's assumed that the function is declared inside the Script defined above.
 	Call string `json:"call"`
 	// Inline LUA code instead of loading from a path via script.
-	Code string `json:"code"`
+	Code string `json:"code,omitempty"`
 	// If these keys are matched, the fields are converted to integer.
 	// If more than one key, delimit by space.
 	// Note that starting from Fluent Bit v1.6 integer data types are preserved

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml

@@ -345,8 +345,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml

@@ -345,8 +345,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.

charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.Kubernetes -}}
+{{- if .Values.fluentbit.enable -}}
+{{- if .Values.fluentbit.filter.multiline.enable -}}
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterFilter
+metadata:
+  name: multiline
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  match: kube.*
+  filters:
+  - multiline:
+      keyContent: {{ .Values.fluentbit.filter.multiline.keyContent | quote }}
+      emitterMemBufLimit: {{ .Values.fluentbit.filter.multiline.emitterMemBufLimit }}
+      parser: "{{- join "," .Values.fluentbit.filter.multiline.parsers -}}"
+{{- end }}
+{{- end }}
+{{- end }}

charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml

@@ -20,6 +20,9 @@ spec:
     {{- else if eq .Values.containerRuntime "crio" }}
     parser: cri
     {{- end }}
+    {{- if .Values.fluentbit.input.tail.multilineParser }}
+    multilineParser: {{ .Values.fluentbit.input.tail.multilineParser | quote }}
+    {{- end }}
     refreshIntervalSeconds: {{ .Values.fluentbit.input.tail.refreshIntervalSeconds }}
     memBufLimit: {{ .Values.fluentbit.input.tail.memBufLimit }}
     {{- if .Values.fluentbit.input.tail.bufferMaxSize }}

charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml

@@ -0,0 +1,30 @@
+{{- if .Values.Kubernetes -}}
+{{- if .Values.fluentbit.enable -}}
+{{- if .Values.fluentbit.parsers.javaMultiline.enable -}}
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterMultilineParser
+metadata:
+  name: java-multiline
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  type: "regex"
+  flushTimeout: 1000
+  keyContent: "log"
+  rules:
+    - start: "start_state"
+      regex: '/^\[?(\d+\-\d+\-\d+ \d+\:\d+\:\d+(\.\d+)?)\]? /'
+      next: "cont"
+    - start: "cont"
+      regex: '/^com\..*/'
+      next: "cont"
+    - start: "cont"
+      regex: '/^\s+.*/'
+      next: "cont"
+    - start: "cont"
+      regex: '/^Caused.*$/'
+      next: "cont"
+{{- end }}
+{{- end }}
+{{- end }}

charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml

@@ -8,7 +8,9 @@ metadata:
     app.kubernetes.io/name: fluent-bit
 spec:
   service:
-    parsersFile: parsers.conf
+    parsersFiles:
+      - /fluent-bit/config/parsers.conf
+      - /fluent-bit/config/parsers_multiline.conf
     httpServer: true
   {{- if .Values.fluentbit.logLevel }}
     logLevel: {{ .Values.fluentbit.logLevel }}
@@ -29,5 +31,8 @@ spec:
   outputSelector:
     matchLabels:
       fluentbit.fluent.io/enabled: "true"
+  multilineParserSelector:
+    matchLabels:
+      fluentbit.fluent.io/enabled: "true"
 {{- end }}
 {{- end }}

charts/fluent-operator/values.yaml

@@ -183,6 +183,8 @@ fluentbit:
       # Use storageType as "filesystem" if you want to use filesystem as the buffering mechanism for tail input.
       storageType: memory
       pauseOnChunksOverlimit: "off"
+      # multiline.parser 
+      # multilineParser: "docker, cri"
     systemd:
       enable: true
       systemdFilter:
@@ -319,6 +321,17 @@ fluentbit:
   # Configure the default filters in FluentBit.
   # The `filter` will filter and parse the collected log information and output the logs into a uniform format. You can choose whether to turn this on or not.
   filter:
+    multiline:
+      enable: false
+      keyContent: log
+      # emitterMemBufLimit 120 (MB)
+      emitterMemBufLimit: 120 
+      parsers:
+        - go
+        - python
+        - java
+        #  use custom multiline parser need set .Values.parsers.javaMultiline.enable = true
+        # - java-multiline
     kubernetes:
       enable: true
       labels: false
@@ -342,6 +355,11 @@ fluentbit:
   # removes the hostPath mounts for varlibcontainers, varlogs and systemd.
   disableLogVolumes: false
 
+  parsers:
+    javaMultiline:
+    # use in filter for parser generic springboot multiline log format
+      enable: false
+
 fluentd:
   # Installs a sub chart carrying the CRDs for the fluentd controller. The sub chart is enabled by default.
   crdsEnable: true

config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml

@@ -345,8 +345,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.

config/crd/bases/fluentbit.fluent.io_filters.yaml

@@ -345,8 +345,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.

config/samples/fluentbit_v1alpha2_clustermultilineparser.yaml

@@ -0,0 +1,18 @@
+apiVersion: fluentbit.fluent.io/v1alpha2
+kind: ClusterMultilineParser
+metadata:
+  name: java-multiline
+  labels:
+    fluentbit.fluent.io/enabled: "true"
+    fluentbit.fluent.io/component: logging
+spec:
+  type: "regex"
+  flushTimeout: 1000
+  keyContent: "log"
+  rules:
+  - start: "start_state"
+    regex: '/\[?(\d+\-\d+\-\d+ \d+\:\d+\:\d+(\.\d+)?)\]? /'
+    next: "cont"
+  - start: "cont"
+    regex: '/^(?!\[?\d+\-\d+\-\d+).*/'
+    next: "cont"

manifests/setup/fluent-operator-crd.yaml

@@ -343,8 +343,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.
@@ -11438,8 +11436,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.

manifests/setup/setup.yaml

@@ -343,8 +343,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.
@@ -11438,8 +11436,6 @@ spec:
                           type: array
                       required:
                       - call
-                      - code
-                      - script
                       type: object
                     modify:
                       description: Modify defines Modify Filter configuration.