DB: improve recreation behavior

cmd/db.go

@@ -121,8 +121,9 @@ func FlattenDB() *cobra.Command {
 
 			log.Printf("Opening database at %s...", dbPath)
 			config := core.Config{
-				BadgerConfig:      badger.DefaultOptions(dbPath),
-				GcIntervalSeconds: 60 * 60 * 24 * 365, // We don't want GC to be running during the flattening
+				RecreateOnCorruption: false,
+				BadgerConfig:         badger.DefaultOptions(dbPath),
+				GcIntervalSeconds:    60 * 60 * 24 * 365, // We don't want GC to be running during the flattening
 			}
 			db, err := core.NewDB(config, codec.NewTinypackCodec())
 			if err != nil {
@@ -150,8 +151,9 @@ func FlattenDB() *cobra.Command {
 // dbView opens db in read-only mode and calls fn with ViewTxn
 func dbView(dbPath string, fn func(txn *core.ViewTxn) error) error {
 	config := core.Config{
-		BadgerConfig:      badger.DefaultOptions(dbPath).WithLogger(nil).WithReadOnly(true),
-		GcIntervalSeconds: 10,
+		RecreateOnCorruption: false,
+		BadgerConfig:         badger.DefaultOptions(dbPath).WithLogger(nil).WithReadOnly(true),
+		GcIntervalSeconds:    10,
 	}
 	db, err := core.NewDB(config, codec.NewTinypackCodec())
 	if err != nil {

db/badger/config.go

@@ -28,11 +28,12 @@ func defaultConfig() *Config {
 	badgerOptions.Logger = NewBadgerLogger(log.Log())
 	return &Config{
 		Core: core.Config{
-			MaxScanIterators:   defaultLogMaxScanIterators,
-			ScanRangeThreshold: defaultLogScanRangeThreshold,
-			FilterTtlMinutes:   defaultLogFilterTtlMinutes,
-			GcIntervalSeconds:  defaultGcIntervalSeconds,
-			BadgerConfig:       badgerOptions,
+			MaxScanIterators:     defaultLogMaxScanIterators,
+			ScanRangeThreshold:   defaultLogScanRangeThreshold,
+			FilterTtlMinutes:     defaultLogFilterTtlMinutes,
+			GcIntervalSeconds:    defaultGcIntervalSeconds,
+			RecreateOnCorruption: true,
+			BadgerConfig:         badgerOptions,
 		},
 	}
 }

db/badger/core/config.go

@@ -3,9 +3,10 @@ package core
 import "github.com/dgraph-io/badger/v3"
 
 type Config struct {
-	MaxScanIterators   uint           `mapstructure:"maxScanIterators"`
-	ScanRangeThreshold uint           `mapstructure:"scanRangeThreshold"`
-	FilterTtlMinutes   int            `mapstructure:"filterTtlMinutes"`
-	GcIntervalSeconds  int            `mapstructure:"gcIntervalSeconds"`
-	BadgerConfig       badger.Options `mapstructure:"options"`
+	MaxScanIterators     uint           `mapstructure:"maxScanIterators"`
+	ScanRangeThreshold   uint           `mapstructure:"scanRangeThreshold"`
+	FilterTtlMinutes     int            `mapstructure:"filterTtlMinutes"`
+	GcIntervalSeconds    int            `mapstructure:"gcIntervalSeconds"`
+	RecreateOnCorruption bool           `mapstructure:"recreateOnCorruption"`
+	BadgerConfig         badger.Options `mapstructure:"options"`
 }

db/badger/core/core.go

@@ -1,23 +1,26 @@
 package core
 
 import (
-	"github.com/aurora-is-near/relayer2-base/log"
-	"github.com/aurora-is-near/relayer2-base/syncutils"
+	"fmt"
 	"os"
 	"path"
+	"strings"
 	"time"
 
+	"github.com/aurora-is-near/relayer2-base/log"
+	"github.com/aurora-is-near/relayer2-base/syncutils"
+
 	"github.com/dgraph-io/badger/v3"
 )
 
 var bdbPtr syncutils.LockablePtr[badger.DB]
 var gcStop chan bool
 
-func Open(options badger.Options, gcIntervalSeconds int) (*badger.DB, error) {
+func Open(options badger.Options, gcIntervalSeconds int, recreateOnCorruption bool) (*badger.DB, error) {
 	var err error
 	bdb, unlock := bdbPtr.LockIfNil()
 	if unlock != nil {
-		bdb, err = tryOpen(options, gcIntervalSeconds)
+		bdb, err = tryOpen(options, gcIntervalSeconds, recreateOnCorruption)
 		unlock(bdb)
 	}
 	return bdb, err
@@ -37,7 +40,7 @@ func Close() error {
 	return nil
 }
 
-func tryOpen(options badger.Options, gcIntervalSeconds int) (*badger.DB, error) {
+func tryOpen(options badger.Options, gcIntervalSeconds int, recreateOnCorruption bool) (*badger.DB, error) {
 	var err error
 	logger := log.Log()
 
@@ -51,6 +54,17 @@ func tryOpen(options badger.Options, gcIntervalSeconds int) (*badger.DB, error)
 	bdb, err := badger.Open(options)
 	if err != nil {
 		logger.Error().Err(err).Msg("failed to tryOpen database")
+
+		// No proper error class for that:
+		// https://github.com/dgraph-io/badger/blob/c08da0b80769f86aa44366e2da77b5c662810eca/dir_unix.go#L67
+		if strings.Contains(err.Error(), "Cannot acquire directory lock") {
+			return nil, fmt.Errorf("db is busy: %w", err)
+		}
+
+		if !recreateOnCorruption {
+			return nil, err
+		}
+
 		snapshotBaseName := path.Base(options.Dir) + "_" + time.Now().Format("2006-01-02T15-04-05.000000000")
 		snapshotPath := path.Join(path.Dir(options.Dir), snapshotBaseName)
 		logger.Warn().Err(err).Msgf("saving old database snapshot at [%s]", snapshotPath)

db/badger/core/db.go

@@ -17,7 +17,7 @@ type DB struct {
 }
 
 func NewDB(config Config, codec codec.Codec) (*DB, error) {
-	core, err := Open(config.BadgerConfig, config.GcIntervalSeconds)
+	core, err := Open(config.BadgerConfig, config.GcIntervalSeconds, config.RecreateOnCorruption)
 	if err != nil {
 		return nil, err
 	}

db/badger/core/db_test.go

@@ -379,7 +379,7 @@ func initTestDb(t *testing.T) (*DB, *testLogger) {
 
 	opts := badger.DefaultOptions("")
 	opts.InMemory = true
-	core, err := Open(opts, 10)
+	core, err := Open(opts, 10, false)
 	require.NoError(t, err, "DB must tryOpen")
 
 	logger := testLogger{}