Merge pull request #497 from mwdomino/rotate-file

add support for rotating file outputs
openconfig · Aug 7, 2024 · fb7904e · fb7904e
2 parents f70355c + 3a77d39
commit fb7904e
Show file tree

Hide file tree

Showing 3 changed files with 101 additions and 23 deletions.
diff --git a/docs/user_guide/outputs/file_output.md b/docs/user_guide/outputs/file_output.md
@@ -54,9 +54,16 @@ outputs:
     enable-metrics: false
      # list of processors to apply on the message before writing
     event-processors:
+    # file rotation configuration
+    rotation:
+      max-size: 100 # size in megabytes
+      max-age: 30 # max age in days
+      max-backups: 3 # maximum number of old files to store, not counting the current file
+      compress: false # whether or not to enable compression
+
 ```
 
-The file output can be used to write to file on the disk, to stdout or to stderr.
+The file output can be used to write to file on the disk, to stdout or to stderr. Also includes support for rotating files to control disk utilization and maximum age using the `rotation` configuration section.
 
 For a disk file, a file name is required.
 

diff --git a/pkg/outputs/file/file_output.go b/pkg/outputs/file/file_output.go
@@ -50,7 +50,7 @@ func init() {
 // File //
 type File struct {
 	cfg    *Config
-	file   *os.File
+	file   file
 	logger *log.Logger
 	mo     *formatters.MarshalOptions
 	sem    *semaphore.Weighted
@@ -62,22 +62,29 @@ type File struct {
 
 // Config //
 type Config struct {
-	FileName           string   `mapstructure:"filename,omitempty"`
-	FileType           string   `mapstructure:"file-type,omitempty"`
-	Format             string   `mapstructure:"format,omitempty"`
-	Multiline          bool     `mapstructure:"multiline,omitempty"`
-	Indent             string   `mapstructure:"indent,omitempty"`
-	Separator          string   `mapstructure:"separator,omitempty"`
-	SplitEvents        bool     `mapstructure:"split-events,omitempty"`
-	OverrideTimestamps bool     `mapstructure:"override-timestamps,omitempty"`
-	AddTarget          string   `mapstructure:"add-target,omitempty"`
-	TargetTemplate     string   `mapstructure:"target-template,omitempty"`
-	EventProcessors    []string `mapstructure:"event-processors,omitempty"`
-	MsgTemplate        string   `mapstructure:"msg-template,omitempty"`
-	ConcurrencyLimit   int      `mapstructure:"concurrency-limit,omitempty"`
-	EnableMetrics      bool     `mapstructure:"enable-metrics,omitempty"`
-	Debug              bool     `mapstructure:"debug,omitempty"`
-	CalculateLatency   bool     `mapstructure:"calculate-latency,omitempty"`
+	FileName           string          `mapstructure:"filename,omitempty"`
+	FileType           string          `mapstructure:"file-type,omitempty"`
+	Format             string          `mapstructure:"format,omitempty"`
+	Multiline          bool            `mapstructure:"multiline,omitempty"`
+	Indent             string          `mapstructure:"indent,omitempty"`
+	Separator          string          `mapstructure:"separator,omitempty"`
+	SplitEvents        bool            `mapstructure:"split-events,omitempty"`
+	OverrideTimestamps bool            `mapstructure:"override-timestamps,omitempty"`
+	AddTarget          string          `mapstructure:"add-target,omitempty"`
+	TargetTemplate     string          `mapstructure:"target-template,omitempty"`
+	EventProcessors    []string        `mapstructure:"event-processors,omitempty"`
+	MsgTemplate        string          `mapstructure:"msg-template,omitempty"`
+	ConcurrencyLimit   int             `mapstructure:"concurrency-limit,omitempty"`
+	EnableMetrics      bool            `mapstructure:"enable-metrics,omitempty"`
+	Debug              bool            `mapstructure:"debug,omitempty"`
+	CalculateLatency   bool            `mapstructure:"calculate-latency,omitempty"`
+	Rotation           *rotationConfig `mapstructure:"rotation,omitempty"`
+}
+
+type file interface {
+	Close() error
+	Name() string
+	Write([]byte) (int, error)
 }
 
 func (f *File) String() string {
@@ -144,11 +151,15 @@ func (f *File) Init(ctx context.Context, name string, cfg map[string]interface{}
 		f.file = os.Stderr
 	default:
 	CRFILE:
-		f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
-		if err != nil {
-			f.logger.Printf("failed to create file: %v", err)
-			time.Sleep(10 * time.Second)
-			goto CRFILE
+		if f.cfg.Rotation != nil {
+			f.file = newRotatingFile(f.cfg)
+		} else {
+			f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+			if err != nil {
+				f.logger.Printf("failed to create file: %v", err)
+				time.Sleep(10 * time.Second)
+				goto CRFILE
+			}
 		}
 	}
 

diff --git a/pkg/outputs/file/rotating_file.go b/pkg/outputs/file/rotating_file.go
@@ -0,0 +1,60 @@
+package file
+
+import (
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+// RotationConfig manages configuration around file rotation
+type rotationConfig struct {
+	MaxSize    int  `mapstructure:"max-size,omitempty"`
+	MaxBackups int  `mapstructure:"max-backups,omitempty"`
+	MaxAge     int  `mapstructure:"max-age,omitempty"`
+	Compress   bool `mapstructure:"compress,omitempty"`
+}
+
+func (r *rotationConfig) SetDefaults() {
+	if r.MaxSize == 0 {
+		r.MaxSize = 100
+	}
+	if r.MaxBackups == 0 {
+		r.MaxBackups = 3
+	}
+
+	if r.MaxAge == 0 {
+		r.MaxAge = 30
+	}
+}
+
+type rotatingFile struct {
+	l *lumberjack.Logger
+}
+
+// newRotatingFile initialize the lumberjack instance
+func newRotatingFile(cfg *Config) *rotatingFile {
+	cfg.Rotation.SetDefaults()
+
+	lj := lumberjack.Logger{
+		Filename:   cfg.FileName,
+		MaxSize:    cfg.Rotation.MaxSize,
+		MaxBackups: cfg.Rotation.MaxBackups,
+		MaxAge:     cfg.Rotation.MaxAge,
+		Compress:   cfg.Rotation.Compress,
+	}
+
+	return &rotatingFile{l: &lj}
+}
+
+// Close closes the file
+func (r *rotatingFile) Close() error {
+	return r.l.Close()
+}
+
+// Name returns the name of the file
+func (r *rotatingFile) Name() string {
+	return r.l.Filename
+}
+
+// Write implements io.Writer
+func (r *rotatingFile) Write(b []byte) (int, error) {
+	return r.l.Write(b)
+}