From 51ad7fd9975e2b8a53fe3e0bfd41e6999b4446fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Mon, 26 Aug 2024 16:19:18 +0200 Subject: [PATCH] feat: Expose lua filter type_array_key parameter (#1323) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Expose lua filter type_array_key parameter The lua filter was missing a parameter `type_array_key`. I've copied the parameter description into the type comment from the fluent-bit docs. Ran `make generate manifests docs-update verify build`. Signed-off-by: Zoltán Reegn * fix var name Signed-off-by: Zoltán Reegn --------- Signed-off-by: Zoltán Reegn --- .../fluentbit/v1alpha2/plugins/filter/lua_types.go | 10 +++++++++- .../plugins/filter/zz_generated.deepcopy.go | 5 +++++ .../crds/fluentbit.fluent.io_clusterfilters.yaml | 7 +++++++ .../crds/fluentbit.fluent.io_filters.yaml | 7 +++++++ .../bases/fluentbit.fluent.io_clusterfilters.yaml | 7 +++++++ config/crd/bases/fluentbit.fluent.io_filters.yaml | 7 +++++++ docs/plugins/fluentbit/filter/lua.md | 1 + docs/plugins/fluentbit/output/open_telemetry.md | 2 +- manifests/setup/fluent-operator-crd.yaml | 14 ++++++++++++++ manifests/setup/setup.yaml | 14 ++++++++++++++ 10 files changed, 72 insertions(+), 2 deletions(-) diff --git a/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go b/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go index 0911ada38..5aee10c60 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go @@ -28,6 +28,9 @@ type Lua struct { // Note that starting from Fluent Bit v1.6 integer data types are preserved // and not converted to double as in previous versions. TypeIntKey []string `json:"typeIntKey,omitempty"` + // If these keys are matched, the fields are handled as array. If more than + // one key, delimit by space. It is useful the array can be empty. + TypeArrayKey []string `json:"typeArrayKey,omitempty"` // If enabled, Lua script will be executed in protected mode. // It prevents to crash when invalid Lua script is executed. Default is true. ProtectedMode *bool `json:"protectedMode,omitempty"` @@ -51,7 +54,7 @@ func (l *Lua) Params(_ plugins.SecretLoader) (*params.KVs, error) { if l.Code != "" { var singleLineLua string = "" - var lineTrim = "" + lineTrim := "" for _, line := range strings.Split(strings.TrimSuffix(l.Code, "\n"), "\n") { lineTrim = strings.TrimSpace(line) if lineTrim != "" { @@ -76,6 +79,10 @@ func (l *Lua) Params(_ plugins.SecretLoader) (*params.KVs, error) { kvs.Insert("type_int_key", strings.Join(l.TypeIntKey, " ")) } + if l.TypeArrayKey != nil && len(l.TypeArrayKey) > 0 { + kvs.Insert("type_array_key", strings.Join(l.TypeArrayKey, " ")) + } + if l.ProtectedMode != nil { kvs.Insert("protected_mode", strconv.FormatBool(*l.ProtectedMode)) } @@ -83,5 +90,6 @@ func (l *Lua) Params(_ plugins.SecretLoader) (*params.KVs, error) { if l.TimeAsTable { kvs.Insert("time_as_table", "true") } + return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go index e4d52d9b2..a0563cf45 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go @@ -335,6 +335,11 @@ func (in *Lua) DeepCopyInto(out *Lua) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TypeArrayKey != nil { + in, out := &in.TypeArrayKey, &out.TypeArrayKey + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.ProtectedMode != nil { in, out := &in.ProtectedMode, &out.ProtectedMode *out = new(bool) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index bbf110f19..af7c84a7e 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -464,6 +464,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index 9550efa28..35dcb1c90 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -464,6 +464,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index bbf110f19..af7c84a7e 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -464,6 +464,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index 9550efa28..35dcb1c90 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -464,6 +464,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. diff --git a/docs/plugins/fluentbit/filter/lua.md b/docs/plugins/fluentbit/filter/lua.md index e16e30cc6..b9ecb4af6 100644 --- a/docs/plugins/fluentbit/filter/lua.md +++ b/docs/plugins/fluentbit/filter/lua.md @@ -9,5 +9,6 @@ The Lua Filter allows you to modify the incoming records using custom Lua Script | call | Lua function name that will be triggered to do filtering. It's assumed that the function is declared inside the Script defined above. | string | | code | Inline LUA code instead of loading from a path via script. | string | | typeIntKey | If these keys are matched, the fields are converted to integer. If more than one key, delimit by space. Note that starting from Fluent Bit v1.6 integer data types are preserved and not converted to double as in previous versions. | []string | +| typeArrayKey | If these keys are matched, the fields are handled as array. If more than one key, delimit by space. It is useful the array can be empty. | []string | | protectedMode | If enabled, Lua script will be executed in protected mode. It prevents to crash when invalid Lua script is executed. Default is true. | *bool | | timeAsTable | By default when the Lua script is invoked, the record timestamp is passed as a Floating number which might lead to loss precision when the data is converted back. If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. | bool | diff --git a/docs/plugins/fluentbit/output/open_telemetry.md b/docs/plugins/fluentbit/output/open_telemetry.md index db1750e73..064271ed3 100644 --- a/docs/plugins/fluentbit/output/open_telemetry.md +++ b/docs/plugins/fluentbit/output/open_telemetry.md @@ -15,7 +15,7 @@ The OpenTelemetry plugin allows you to take logs, metrics, and traces from Fluen | tracesUri | Specify an optional HTTP URI for the target web server listening for traces, e.g: /v1/traces | string | | header | Add a HTTP header key/value pair. Multiple headers can be set. | map[string]string | | logResponsePayload | Log the response payload within the Fluent Bit log. | *bool | -| logsBodyKeyAttributes | Adds unmatched keys as attributes when true. | *bool | | addLabel | This allows you to add custom labels to all metrics exposed through the OpenTelemetry exporter. You may have multiple of these fields. | map[string]string | +| logsBodyKeyAttributes | If true, remaining unmatched keys are added as attributes. | *bool | | tls | | *[plugins.TLS](../tls.md) | | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 07c4c64bc..294c9addf 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -463,6 +463,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. @@ -14959,6 +14966,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 46d0d5757..404a92c9c 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -463,6 +463,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer. @@ -14959,6 +14966,13 @@ spec: If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. type: boolean + typeArrayKey: + description: |- + If these keys are matched, the fields are handled as array. If more than + one key, delimit by space. It is useful the array can be empty. + items: + type: string + type: array typeIntKey: description: |- If these keys are matched, the fields are converted to integer.