From 4b6a6116deaea8141b3a86a8ab8268b16a09a0e3 Mon Sep 17 00:00:00 2001 From: Miguel Angel Rojo Fernandez Date: Fri, 10 Jan 2025 15:28:35 +0000 Subject: [PATCH 1/3] added counter for db locking --- muxdb/metrics.go | 5 ++++- muxdb/muxdb.go | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/muxdb/metrics.go b/muxdb/metrics.go index 65ead18fb..9d7b0f4e7 100644 --- a/muxdb/metrics.go +++ b/muxdb/metrics.go @@ -11,4 +11,7 @@ import ( "github.com/vechain/thor/v2/metrics" ) -var metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"}) +var ( + metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"}) + metricLeveldbLock = metrics.LazyLoadCounterVec("leveldb_lock_count", []string{"event"}) +) diff --git a/muxdb/muxdb.go b/muxdb/muxdb.go index a0f00ae2c..46edd5fd3 100644 --- a/muxdb/muxdb.go +++ b/muxdb/muxdb.go @@ -10,6 +10,8 @@ package muxdb import ( "context" "encoding/json" + "os" + "syscall" "github.com/syndtr/goleveldb/leveldb" dberrors "github.com/syndtr/goleveldb/leveldb/errors" @@ -62,6 +64,24 @@ type MuxDB struct { trieBackend *backend } +// Adds metrics if the error is due to file/db lock. +func addMetricsIfLocked(err error, event string) { + if err == nil { + return + } + + if pathErr, ok := err.(*os.PathError); ok { + // Eventually calls to openFileNoLog https://go.dev/src/os/file_unix.go + if errno, ok := pathErr.Err.(syscall.Errno); ok && errno == syscall.EWOULDBLOCK { + metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) + } + } else if err == syscall.EWOULDBLOCK { // setFileLock https://github.com/vechain/goleveldb/blob/master/leveldb/storage/file_storage_unix.go#L50 + metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) + } else if err == storage.ErrLocked { // If using sessions + metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) + } +} + // Open opens or creates DB at the given path. func Open(path string, options *Options) (*MuxDB, error) { // prepare leveldb options @@ -82,9 +102,12 @@ func Open(path string, options *Options) (*MuxDB, error) { // open leveldb ldb, err := leveldb.OpenFile(path, &ldbOpts) + addMetricsIfLocked(err, "open-file") if _, corrupted := err.(*dberrors.ErrCorrupted); corrupted { ldb, err = leveldb.RecoverFile(path, &ldbOpts) + addMetricsIfLocked(err, "recover-file") } + if err != nil { return nil, err } @@ -120,7 +143,8 @@ func Open(path string, options *Options) (*MuxDB, error) { // NewMem creates a memory-backed DB. func NewMem() *MuxDB { storage := storage.NewMemStorage() - ldb, _ := leveldb.Open(storage, nil) + ldb, err := leveldb.Open(storage, nil) + addMetricsIfLocked(err, "open-memory-backed-db") engine := engine.NewLevelEngine(ldb) return &MuxDB{ From b2462eb2b29a0e87a74938ece7331189e2c58952 Mon Sep 17 00:00:00 2001 From: Miguel Angel Rojo Fernandez Date: Fri, 10 Jan 2025 15:39:25 +0000 Subject: [PATCH 2/3] added counter for db locking --- cmd/thor/node/node.go | 2 ++ muxdb/muxdb.go | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/thor/node/node.go b/cmd/thor/node/node.go index 251e421b0..1f87c5b3f 100644 --- a/cmd/thor/node/node.go +++ b/cmd/thor/node/node.go @@ -28,6 +28,7 @@ import ( "github.com/vechain/thor/v2/consensus" "github.com/vechain/thor/v2/log" "github.com/vechain/thor/v2/logdb" + "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/packer" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/thor" @@ -235,6 +236,7 @@ func (n *Node) txStashLoop(ctx context.Context) { db, err := leveldb.OpenFile(n.txStashPath, nil) if err != nil { + muxdb.AddMetricsIfLocked(err, "open-file-tx-stash") logger.Error("create tx stash", "err", err) return } diff --git a/muxdb/muxdb.go b/muxdb/muxdb.go index 46edd5fd3..a1fd59992 100644 --- a/muxdb/muxdb.go +++ b/muxdb/muxdb.go @@ -65,7 +65,7 @@ type MuxDB struct { } // Adds metrics if the error is due to file/db lock. -func addMetricsIfLocked(err error, event string) { +func AddMetricsIfLocked(err error, event string) { if err == nil { return } @@ -102,10 +102,10 @@ func Open(path string, options *Options) (*MuxDB, error) { // open leveldb ldb, err := leveldb.OpenFile(path, &ldbOpts) - addMetricsIfLocked(err, "open-file") + AddMetricsIfLocked(err, "open-file") if _, corrupted := err.(*dberrors.ErrCorrupted); corrupted { ldb, err = leveldb.RecoverFile(path, &ldbOpts) - addMetricsIfLocked(err, "recover-file") + AddMetricsIfLocked(err, "recover-file") } if err != nil { @@ -144,7 +144,7 @@ func Open(path string, options *Options) (*MuxDB, error) { func NewMem() *MuxDB { storage := storage.NewMemStorage() ldb, err := leveldb.Open(storage, nil) - addMetricsIfLocked(err, "open-memory-backed-db") + AddMetricsIfLocked(err, "open-memory-backed-db") engine := engine.NewLevelEngine(ldb) return &MuxDB{ From abee6eb791db888ad99d6653de05ae9581c57c94 Mon Sep 17 00:00:00 2001 From: Miguel Angel Rojo Fernandez Date: Fri, 10 Jan 2025 15:46:31 +0000 Subject: [PATCH 3/3] refactor --- muxdb/muxdb.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/muxdb/muxdb.go b/muxdb/muxdb.go index a1fd59992..9bc35d14e 100644 --- a/muxdb/muxdb.go +++ b/muxdb/muxdb.go @@ -70,14 +70,27 @@ func AddMetricsIfLocked(err error, event string) { return } - if pathErr, ok := err.(*os.PathError); ok { + isLockError := false + + switch e := err.(type) { + case *os.PathError: // Eventually calls to openFileNoLog https://go.dev/src/os/file_unix.go - if errno, ok := pathErr.Err.(syscall.Errno); ok && errno == syscall.EWOULDBLOCK { - metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) + if errno, ok := e.Err.(syscall.Errno); ok && errno == syscall.EWOULDBLOCK { + isLockError = true } - } else if err == syscall.EWOULDBLOCK { // setFileLock https://github.com/vechain/goleveldb/blob/master/leveldb/storage/file_storage_unix.go#L50 - metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) - } else if err == storage.ErrLocked { // If using sessions + case syscall.Errno: + // setFileLock https://github.com/vechain/goleveldb/blob/master/leveldb/storage/file_storage_unix.go#L50 + if e == syscall.EWOULDBLOCK { + isLockError = true + } + case error: + // If using sessions + if e == storage.ErrLocked { + isLockError = true + } + } + + if isLockError { metricLeveldbLock().AddWithLabel(1, map[string]string{"event": event}) } }