Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add json format support for log export via faro receiver #2276

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ Main (unreleased)
- Add perf_schema quantile columns to collector

- Live Debugging button should appear in UI only for supported components (@ravishankar15)
- Add json format support for log export via faro receiver (@ravishankar15)
- Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15)
- Add `ignore_older_than` option for local.file_match (@ravishankar15)
- Add livedebugging support for `discover.relabel` (@ravishankar15)
Expand Down
8 changes: 8 additions & 0 deletions docs/sources/reference/components/faro/faro.receiver.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@ The following arguments are supported:
Name | Type | Description | Default | Required
-------------------|---------------|----------------------------------------------|---------|---------
`extra_log_labels` | `map(string)` | Extra labels to attach to emitted log lines. | `{}` | no
`log_format` | `string` | Export format for the logs. | `logfmt`| no

### Log format

The following strings are recognized as valid log line formats:
ravishankar15 marked this conversation as resolved.
Show resolved Hide resolved

* `"logfmt"`: Export logs as [logfmt](https://brandur.org/logfmt) lines.
* `"json"`: Export logs as JSON objects.

## Blocks

Expand Down
34 changes: 34 additions & 0 deletions internal/component/faro/receiver/arguments.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package receiver

import (
"encoding"
"fmt"
"time"

"github.com/alecthomas/units"
Expand All @@ -13,6 +15,7 @@ import (
// Arguments configures the app_agent_receiver component.
type Arguments struct {
LogLabels map[string]string `alloy:"extra_log_labels,attr,optional"`
LogFormat LogFormat `alloy:"log_format,attr,optional"`

Server ServerArguments `alloy:"server,block,optional"`
SourceMaps SourceMapsArguments `alloy:"sourcemaps,block,optional"`
Expand All @@ -23,6 +26,7 @@ var _ syntax.Defaulter = (*Arguments)(nil)

// SetToDefault applies default settings.
func (args *Arguments) SetToDefault() {
args.LogFormat = FormatDefault
args.Server.SetToDefault()
args.SourceMaps.SetToDefault()
}
Expand Down Expand Up @@ -93,3 +97,33 @@ type OutputArguments struct {
Logs []loki.LogsReceiver `alloy:"logs,attr,optional"`
Traces []otelcol.Consumer `alloy:"traces,attr,optional"`
}

type LogFormat string

const (
FormatLogfmt LogFormat = "logfmt"
FormatJSON LogFormat = "json"

FormatDefault = FormatLogfmt
)

var (
_ encoding.TextMarshaler = FormatDefault
_ encoding.TextUnmarshaler = (*LogFormat)(nil)
)

func (ll LogFormat) MarshalText() (text []byte, err error) {
return []byte(ll), nil
}

func (ll *LogFormat) UnmarshalText(text []byte) error {
switch LogFormat(text) {
case "":
*ll = FormatDefault
case FormatLogfmt, FormatJSON:
*ll = LogFormat(text)
default:
return fmt.Errorf("unrecognized log format %q", string(text))
}
return nil
}
19 changes: 17 additions & 2 deletions internal/component/faro/receiver/exporters.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package receiver

import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
Expand Down Expand Up @@ -83,6 +84,7 @@ func (exp *metricsExporter) Export(ctx context.Context, p payload.Payload) error
type logsExporter struct {
log log.Logger
sourceMaps sourceMapsStore
format LogFormat

receiversMut sync.RWMutex
receivers []loki.LogsReceiver
Expand All @@ -93,10 +95,11 @@ type logsExporter struct {

var _ exporter = (*logsExporter)(nil)

func newLogsExporter(log log.Logger, sourceMaps sourceMapsStore) *logsExporter {
func newLogsExporter(log log.Logger, sourceMaps sourceMapsStore, format LogFormat) *logsExporter {
return &logsExporter{
log: log,
sourceMaps: sourceMaps,
format: format,
}
}

Expand Down Expand Up @@ -157,7 +160,19 @@ func (exp *logsExporter) sendKeyValsToLogsPipeline(ctx context.Context, kv *payl
)
exp.receiversMut.RUnlock()

line, err := logfmt.MarshalKeyvals(payload.KeyValToInterfaceSlice(kv)...)
var (
line []byte
err error
)
switch exp.format {
case FormatLogfmt:
line, err = logfmt.MarshalKeyvals(payload.KeyValToInterfaceSlice(kv)...)
case FormatJSON:
line, err = json.Marshal(payload.KeyValToInterfaceMap(kv))
default:
line, err = logfmt.MarshalKeyvals(payload.KeyValToInterfaceSlice(kv)...)
}

if err != nil {
level.Error(exp.log).Log("msg", "failed to logfmt a frontend log event", "err", err)
return err
Expand Down
Loading
Loading