diff --git a/cmd/thor/node/node.go b/cmd/thor/node/node.go index 251e421b0..1f87c5b3f 100644 --- a/cmd/thor/node/node.go +++ b/cmd/thor/node/node.go @@ -28,6 +28,7 @@ import ( "github.com/vechain/thor/v2/consensus" "github.com/vechain/thor/v2/log" "github.com/vechain/thor/v2/logdb" + "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/packer" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/thor" @@ -235,6 +236,7 @@ func (n *Node) txStashLoop(ctx context.Context) { db, err := leveldb.OpenFile(n.txStashPath, nil) if err != nil { + muxdb.AddMetricsIfLocked(err, "open-file-tx-stash") logger.Error("create tx stash", "err", err) return } diff --git a/muxdb/metrics.go b/muxdb/metrics.go index 65ead18fb..9d7b0f4e7 100644 --- a/muxdb/metrics.go +++ b/muxdb/metrics.go @@ -11,4 +11,7 @@ import ( "github.com/vechain/thor/v2/metrics" ) -var metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"}) +var ( + metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"}) + metricLeveldbLock = metrics.LazyLoadCounterVec("leveldb_lock_count", []string{"event"}) +) diff --git a/muxdb/muxdb.go b/muxdb/muxdb.go index a0f00ae2c..9bc35d14e 100644 --- a/muxdb/muxdb.go +++ b/muxdb/muxdb.go @@ -10,6 +10,8 @@ package muxdb import ( "context" "encoding/json" + "os" + "syscall" "github.com/syndtr/goleveldb/leveldb" dberrors "github.com/syndtr/goleveldb/leveldb/errors" @@ -62,6 +64,37 @@ type MuxDB struct { trieBackend *backend } +// Adds metrics if the error is due to file/db lock. +func AddMetricsIfLocked(err error, event string) { + if err == nil { + return + } + + isLockError := false + + switch e := err.(type) { + case *os.PathError: + // Eventually calls to openFileNoLog https://go.dev/src/os/file_unix.go + if errno, ok := e.Err.(syscall.Errno); ok && errno == syscall.EWOULDBLOCK { + isLockError = true + } + case syscall.Errno: + // setFileLock https://github.com/vechain/goleveldb/blob/master/leveldb/storage/file_storage_unix.go#L50 + if e == syscall.EWOULDBLOCK { + isLockError = true + } + case error: + // If using sessions + if e == storage.ErrLocked { + isLockError = true + } + } + + if isLockError { + metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) + } +} + // Open opens or creates DB at the given path. func Open(path string, options *Options) (*MuxDB, error) { // prepare leveldb options @@ -82,9 +115,12 @@ func Open(path string, options *Options) (*MuxDB, error) { // open leveldb ldb, err := leveldb.OpenFile(path, &ldbOpts) + AddMetricsIfLocked(err, "open-file") if _, corrupted := err.(*dberrors.ErrCorrupted); corrupted { ldb, err = leveldb.RecoverFile(path, &ldbOpts) + AddMetricsIfLocked(err, "recover-file") } + if err != nil { return nil, err } @@ -120,7 +156,8 @@ func Open(path string, options *Options) (*MuxDB, error) { // NewMem creates a memory-backed DB. func NewMem() *MuxDB { storage := storage.NewMemStorage() - ldb, _ := leveldb.Open(storage, nil) + ldb, err := leveldb.Open(storage, nil) + AddMetricsIfLocked(err, "open-memory-backed-db") engine := engine.NewLevelEngine(ldb) return &MuxDB{