diff --git a/docs/user_guide/outputs/file_output.md b/docs/user_guide/outputs/file_output.md index 0c35a7b4..737a507d 100644 --- a/docs/user_guide/outputs/file_output.md +++ b/docs/user_guide/outputs/file_output.md @@ -54,9 +54,16 @@ outputs: enable-metrics: false # list of processors to apply on the message before writing event-processors: + # file rotation configuration + rotation: + max-size: 100 # size in megabytes + max-age: 30 # max age in days + max-backups: 3 # maximum number of old files to store, not counting the current file + compress: false # whether or not to enable compression + ``` -The file output can be used to write to file on the disk, to stdout or to stderr. +The file output can be used to write to file on the disk, to stdout or to stderr. Also includes support for rotating files to control disk utilization and maximum age using the `rotation` configuration section. For a disk file, a file name is required. diff --git a/pkg/outputs/file/file_output.go b/pkg/outputs/file/file_output.go index 1452bbae..aed8bbd8 100644 --- a/pkg/outputs/file/file_output.go +++ b/pkg/outputs/file/file_output.go @@ -50,7 +50,7 @@ func init() { // File // type File struct { cfg *Config - file *os.File + file file logger *log.Logger mo *formatters.MarshalOptions sem *semaphore.Weighted @@ -62,22 +62,29 @@ type File struct { // Config // type Config struct { - FileName string `mapstructure:"filename,omitempty"` - FileType string `mapstructure:"file-type,omitempty"` - Format string `mapstructure:"format,omitempty"` - Multiline bool `mapstructure:"multiline,omitempty"` - Indent string `mapstructure:"indent,omitempty"` - Separator string `mapstructure:"separator,omitempty"` - SplitEvents bool `mapstructure:"split-events,omitempty"` - OverrideTimestamps bool `mapstructure:"override-timestamps,omitempty"` - AddTarget string `mapstructure:"add-target,omitempty"` - TargetTemplate string `mapstructure:"target-template,omitempty"` - EventProcessors []string `mapstructure:"event-processors,omitempty"` - MsgTemplate string `mapstructure:"msg-template,omitempty"` - ConcurrencyLimit int `mapstructure:"concurrency-limit,omitempty"` - EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` - Debug bool `mapstructure:"debug,omitempty"` - CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` + FileName string `mapstructure:"filename,omitempty"` + FileType string `mapstructure:"file-type,omitempty"` + Format string `mapstructure:"format,omitempty"` + Multiline bool `mapstructure:"multiline,omitempty"` + Indent string `mapstructure:"indent,omitempty"` + Separator string `mapstructure:"separator,omitempty"` + SplitEvents bool `mapstructure:"split-events,omitempty"` + OverrideTimestamps bool `mapstructure:"override-timestamps,omitempty"` + AddTarget string `mapstructure:"add-target,omitempty"` + TargetTemplate string `mapstructure:"target-template,omitempty"` + EventProcessors []string `mapstructure:"event-processors,omitempty"` + MsgTemplate string `mapstructure:"msg-template,omitempty"` + ConcurrencyLimit int `mapstructure:"concurrency-limit,omitempty"` + EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` + Debug bool `mapstructure:"debug,omitempty"` + CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` + Rotation *rotationConfig `mapstructure:"rotation,omitempty"` +} + +type file interface { + Close() error + Name() string + Write([]byte) (int, error) } func (f *File) String() string { @@ -144,11 +151,15 @@ func (f *File) Init(ctx context.Context, name string, cfg map[string]interface{} f.file = os.Stderr default: CRFILE: - f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - f.logger.Printf("failed to create file: %v", err) - time.Sleep(10 * time.Second) - goto CRFILE + if f.cfg.Rotation != nil { + f.file = newRotatingFile(f.cfg) + } else { + f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + f.logger.Printf("failed to create file: %v", err) + time.Sleep(10 * time.Second) + goto CRFILE + } } } diff --git a/pkg/outputs/file/rotating_file.go b/pkg/outputs/file/rotating_file.go new file mode 100644 index 00000000..6b5098db --- /dev/null +++ b/pkg/outputs/file/rotating_file.go @@ -0,0 +1,60 @@ +package file + +import ( + "gopkg.in/natefinch/lumberjack.v2" +) + +// RotationConfig manages configuration around file rotation +type rotationConfig struct { + MaxSize int `mapstructure:"max-size,omitempty"` + MaxBackups int `mapstructure:"max-backups,omitempty"` + MaxAge int `mapstructure:"max-age,omitempty"` + Compress bool `mapstructure:"compress,omitempty"` +} + +func (r *rotationConfig) SetDefaults() { + if r.MaxSize == 0 { + r.MaxSize = 100 + } + if r.MaxBackups == 0 { + r.MaxBackups = 3 + } + + if r.MaxAge == 0 { + r.MaxAge = 30 + } +} + +type rotatingFile struct { + l *lumberjack.Logger +} + +// newRotatingFile initialize the lumberjack instance +func newRotatingFile(cfg *Config) *rotatingFile { + cfg.Rotation.SetDefaults() + + lj := lumberjack.Logger{ + Filename: cfg.FileName, + MaxSize: cfg.Rotation.MaxSize, + MaxBackups: cfg.Rotation.MaxBackups, + MaxAge: cfg.Rotation.MaxAge, + Compress: cfg.Rotation.Compress, + } + + return &rotatingFile{l: &lj} +} + +// Close closes the file +func (r *rotatingFile) Close() error { + return r.l.Close() +} + +// Name returns the name of the file +func (r *rotatingFile) Name() string { + return r.l.Filename +} + +// Write implements io.Writer +func (r *rotatingFile) Write(b []byte) (int, error) { + return r.l.Write(b) +}