Metrics (Database locking): Added counter

cmd/thor/node/node.go

@@ -28,6 +28,7 @@ import (
 	"github.com/vechain/thor/v2/consensus"
 	"github.com/vechain/thor/v2/log"
 	"github.com/vechain/thor/v2/logdb"
+	"github.com/vechain/thor/v2/muxdb"
 	"github.com/vechain/thor/v2/packer"
 	"github.com/vechain/thor/v2/state"
 	"github.com/vechain/thor/v2/thor"
@@ -235,6 +236,7 @@ func (n *Node) txStashLoop(ctx context.Context) {
 
 	db, err := leveldb.OpenFile(n.txStashPath, nil)
 	if err != nil {
+		muxdb.AddMetricsIfLocked(err, "open-file-tx-stash")
 		logger.Error("create tx stash", "err", err)
 		return
 	}

muxdb/metrics.go

@@ -11,4 +11,7 @@ import (
 	"github.com/vechain/thor/v2/metrics"
 )
 
-var metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"})
+var (
+	metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"})
+	metricLeveldbLock  = metrics.LazyLoadCounterVec("leveldb_lock_count", []string{"event"})
+)

muxdb/muxdb.go

@@ -10,6 +10,8 @@ package muxdb
 import (
 	"context"
 	"encoding/json"
+	"os"
+	"syscall"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	dberrors "github.com/syndtr/goleveldb/leveldb/errors"
@@ -62,6 +64,37 @@ type MuxDB struct {
 	trieBackend *backend
 }
 
+// Adds metrics if the error is due to file/db lock.
+func AddMetricsIfLocked(err error, event string) {
+	if err == nil {
+		return
+	}
+
+	isLockError := false
+
+	switch e := err.(type) {
+	case *os.PathError:
+		// Eventually calls to openFileNoLog https://go.dev/src/os/file_unix.go
+		if errno, ok := e.Err.(syscall.Errno); ok && errno == syscall.EWOULDBLOCK {
+			isLockError = true
+		}
+	case syscall.Errno:
+		// setFileLock https://github.com/vechain/goleveldb/blob/master/leveldb/storage/file_storage_unix.go#L50
+		if e == syscall.EWOULDBLOCK {
+			isLockError = true
+		}
+	case error:
+		// If using sessions
+		if e == storage.ErrLocked {
+			isLockError = true
+		}
+	}
+
+	if isLockError {
+		metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event})
+	}
+}
+
 // Open opens or creates DB at the given path.
 func Open(path string, options *Options) (*MuxDB, error) {
 	// prepare leveldb options
@@ -82,9 +115,12 @@ func Open(path string, options *Options) (*MuxDB, error) {
 
 	// open leveldb
 	ldb, err := leveldb.OpenFile(path, &ldbOpts)
+	AddMetricsIfLocked(err, "open-file")
 	if _, corrupted := err.(*dberrors.ErrCorrupted); corrupted {
 		ldb, err = leveldb.RecoverFile(path, &ldbOpts)
+		AddMetricsIfLocked(err, "recover-file")
 	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -120,7 +156,8 @@ func Open(path string, options *Options) (*MuxDB, error) {
 // NewMem creates a memory-backed DB.
 func NewMem() *MuxDB {
 	storage := storage.NewMemStorage()
-	ldb, _ := leveldb.Open(storage, nil)
+	ldb, err := leveldb.Open(storage, nil)
+	AddMetricsIfLocked(err, "open-memory-backed-db")
 
 	engine := engine.NewLevelEngine(ldb)
 	return &MuxDB{