feat: Expose lua filter type_array_key parameter (#1323)

* feat: Expose lua filter type_array_key parameter The lua filter was missing a parameter `type_array_key`. I've copied the parameter description into the type comment from the fluent-bit docs. Ran `make generate manifests docs-update verify build`. Signed-off-by: Zoltán Reegn <[email protected]> * fix var name Signed-off-by: Zoltán Reegn <[email protected]> --------- Signed-off-by: Zoltán Reegn <[email protected]>
fluent · Aug 26, 2024 · 51ad7fd · 51ad7fd
1 parent 850cb1b
commit 51ad7fd
Show file tree

Hide file tree

Showing 10 changed files with 72 additions and 2 deletions.
diff --git a/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go b/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go
@@ -28,6 +28,9 @@ type Lua struct {
 	// Note that starting from Fluent Bit v1.6 integer data types are preserved
 	// and not converted to double as in previous versions.
 	TypeIntKey []string `json:"typeIntKey,omitempty"`
+	// If these keys are matched, the fields are handled as array. If more than
+	// one key, delimit by space. It is useful the array can be empty.
+	TypeArrayKey []string `json:"typeArrayKey,omitempty"`
 	// If enabled, Lua script will be executed in protected mode.
 	// It prevents to crash when invalid Lua script is executed. Default is true.
 	ProtectedMode *bool `json:"protectedMode,omitempty"`
@@ -51,7 +54,7 @@ func (l *Lua) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 
 	if l.Code != "" {
 		var singleLineLua string = ""
-		var lineTrim = ""
+		lineTrim := ""
 		for _, line := range strings.Split(strings.TrimSuffix(l.Code, "\n"), "\n") {
 			lineTrim = strings.TrimSpace(line)
 			if lineTrim != "" {
@@ -76,12 +79,17 @@ func (l *Lua) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 		kvs.Insert("type_int_key", strings.Join(l.TypeIntKey, " "))
 	}
 
+	if l.TypeArrayKey != nil && len(l.TypeArrayKey) > 0 {
+		kvs.Insert("type_array_key", strings.Join(l.TypeArrayKey, " "))
+	}
+
 	if l.ProtectedMode != nil {
 		kvs.Insert("protected_mode", strconv.FormatBool(*l.ProtectedMode))
 	}
 
 	if l.TimeAsTable {
 		kvs.Insert("time_as_table", "true")
 	}
+
 	return kvs, nil
 }
diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go
diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml
@@ -464,6 +464,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.

diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml
@@ -464,6 +464,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.

diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml
@@ -464,6 +464,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.

diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml
@@ -464,6 +464,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.

diff --git a/docs/plugins/fluentbit/filter/lua.md b/docs/plugins/fluentbit/filter/lua.md
@@ -9,5 +9,6 @@ The Lua Filter allows you to modify the incoming records using custom Lua Script
 | call | Lua function name that will be triggered to do filtering. It's assumed that the function is declared inside the Script defined above. | string |
 | code | Inline LUA code instead of loading from a path via script. | string |
 | typeIntKey | If these keys are matched, the fields are converted to integer. If more than one key, delimit by space. Note that starting from Fluent Bit v1.6 integer data types are preserved and not converted to double as in previous versions. | []string |
+| typeArrayKey | If these keys are matched, the fields are handled as array. If more than one key, delimit by space. It is useful the array can be empty. | []string |
 | protectedMode | If enabled, Lua script will be executed in protected mode. It prevents to crash when invalid Lua script is executed. Default is true. | *bool |
 | timeAsTable | By default when the Lua script is invoked, the record timestamp is passed as a Floating number which might lead to loss precision when the data is converted back. If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. | bool |
diff --git a/docs/plugins/fluentbit/output/open_telemetry.md b/docs/plugins/fluentbit/output/open_telemetry.md
@@ -15,7 +15,7 @@ The OpenTelemetry plugin allows you to take logs, metrics, and traces from Fluen
 | tracesUri | Specify an optional HTTP URI for the target web server listening for traces, e.g: /v1/traces | string |
 | header | Add a HTTP header key/value pair. Multiple headers can be set. | map[string]string |
 | logResponsePayload | Log the response payload within the Fluent Bit log. | *bool |
-| logsBodyKeyAttributes | Adds unmatched keys as attributes when true. | *bool |
 | addLabel | This allows you to add custom labels to all metrics exposed through the OpenTelemetry exporter. You may have multiple of these fields. | map[string]string |
+| logsBodyKeyAttributes | If true, remaining unmatched keys are added as attributes. | *bool |
 | tls |  | *[plugins.TLS](../tls.md) |
 | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking |
diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml
@@ -463,6 +463,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.
@@ -14959,6 +14966,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.

diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml
@@ -463,6 +463,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.
@@ -14959,6 +14966,13 @@ spec:
                             If you desire timestamp precision enabling this option will pass the timestamp as
                             a Lua table with keys sec for seconds since epoch and nsec for nanoseconds.
                           type: boolean
+                        typeArrayKey:
+                          description: |-
+                            If these keys are matched, the fields are handled as array. If more than
+                            one key, delimit by space. It is useful the array can be empty.
+                          items:
+                            type: string
+                          type: array
                         typeIntKey:
                           description: |-
                             If these keys are matched, the fields are converted to integer.