From 109a6fba81b623f110582bdd8f35044a00a3053b Mon Sep 17 00:00:00 2001 From: Miguel Angel Rojo Date: Mon, 20 Jan 2025 16:34:05 +0000 Subject: [PATCH] Metrics (Compaction): Values by level (#936) * Darren/logdb remove leading zeros (#865) * Add LogIndex and TxIndex into logs/event response body (#862) * Thor client (#818) * feat: add thorclient * refactor: remove roundTripper * refactor: change null check * clean: remove commented code * feat: add account revision and pending tx * fix: add licence headers and fix linter issue * refactor: rename package * refactor: change revision type to string * refactor: rename GetLogs and GetTransfers to FilterEvents and FilterTransfers * refactor: change FilterEvents and FilterTransactions request type to EventFilter * Adding common.EventWrapper to handle channel errors * tweak * update rawclient + update account tests * tidy up names * update tests * pr comments * adding raw tx * Tidy up method names and calls * options client * tweaks * pr comments * Update thorclient/common/common.go Co-authored-by: libotony * pr comments * Adding Subscriptions * Pr comments * adjust func orders * pr comments * changing subscribe to use the channel close vs multiple channels * adding go-doc * no error after unsubscribe * pr comments * checking status code is 2xx * fix: change FilterTransfers argument --------- Co-authored-by: otherview Co-authored-by: libotony * Show all issues on lint (#869) * Show all issues on lint * fix lint * fix(docker): using AWS docker repo for trivy (#872) * fix(docker): using AWS docker repo for trivy * fix(docker): using AWS docker repo for trivy * Darren/feat/add subscription cache (#866) * ehancement: create a cache for block based subscriptions * minor: change function names for subscriptions * test: add unit test for message cache * chore: add license headers * refactor: fix up error handling * fix: remove bad test * fix: PR comments * fix: PR comments - remove block cache * refactor(subscriptions): store structs in cache, not bytes * fix(license): add license header * chore(subscriptions): revert unit test changes * enhancement: resolve pr comments to use simplelru * enhancement: resolve pr comments - use id as key * Add additional block tests (#863) * enhancement(logging): leverage trace level (#873) * Add testchain package (#844) * Refactor thor node * thorchain allows insertion of blocks * remove thorNode, added testchain * clean up + comments * adding license headers * adding templating tests for thorclient * Remove test event hacks * remove types * removed chain_builder + added logdb to testchain * pr comments * Update test/testchain/chain.go Co-authored-by: libotony --------- Co-authored-by: libotony * chore(docs): update spec for validator nodes (#875) * chore(docs): update spec for validator nodes * chore(docs): update cores * chore(docs): remove public node stuff * Darren/logdb remove leading zeros (#865) * feat: add new txIndex column to event meta response * test: add convert event test * feat: make txLog and txIndex as optional return params * chore: update swagger with new event optional data * feat: save logIndex in sequence * feat: tweaked bits in sequence * refactor: rename optional log meta field * refactor: comments, yaml and txIndex counts * rebase to master * fix: remove stale struct * add txIndex to returned logdb query * reset to 0 eventCount and transferCount each receipt and write blockId only once * fix lint * rephrase logIndex description in yaml file * refactor: use filter.Option instead of eventFilter.Option * move includeIndexes to api --------- Co-authored-by: otherview Co-authored-by: libotony Co-authored-by: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Co-authored-by: Makis Christou * fix: set range.To to max logDB block (#880) * Update Convert Filter to match BlockMask bit space (#881) * Update Convert Filter to match BlockMask bit space * use exported BlockNumMask * return error in newSequence (#885) * return error in newSequence * revert type change of sequence * adjust sequence to 63bit * fix test --------- Co-authored-by: otherview * add safety guard to the sequence (#887) * add safety guard to the sequence * move bit distribution to tests * chore: update version for mainDB v4 * maindb v4 (#868) * fix(documentation): use absolute links in markdown (#889) * Add benchmark test to node block process (#892) * Add benchmark test to node block process * added file-based storage * use tempdir * update dependency go-ethereum (#895) * chore: update API metrics bucket and endpoint names (#893) * chore: update API metrics bucket and endpoint names * fix: typo & tests * fix: lint * chore: add websocket total counter * fix: txs endpoints names & ws subject * fix: unit tests * chore: standardise naming convention * chore: add websocke duration & http code * chore: add websocke duration & http code * fix: lint issues * fix: sync issues with metrics * chore: update websocket durations bucket * fix: PR comments - use sync.Once * chore: update builtin generation (#896) * chore: update builtin generation * fix: update GHA * Thor client (#818) * feat: add thorclient * refactor: remove roundTripper * refactor: change null check * clean: remove commented code * feat: add account revision and pending tx * fix: add licence headers and fix linter issue * refactor: rename package * refactor: change revision type to string * refactor: rename GetLogs and GetTransfers to FilterEvents and FilterTransfers * refactor: change FilterEvents and FilterTransactions request type to EventFilter * Adding common.EventWrapper to handle channel errors * tweak * update rawclient + update account tests * tidy up names * update tests * pr comments * adding raw tx * Tidy up method names and calls * options client * tweaks * pr comments * Update thorclient/common/common.go Co-authored-by: libotony * pr comments * Adding Subscriptions * Pr comments * adjust func orders * pr comments * changing subscribe to use the channel close vs multiple channels * adding go-doc * no error after unsubscribe * pr comments * checking status code is 2xx * fix: change FilterTransfers argument --------- Co-authored-by: otherview Co-authored-by: libotony * Add testchain package (#844) * Refactor thor node * thorchain allows insertion of blocks * remove thorNode, added testchain * clean up + comments * adding license headers * adding templating tests for thorclient * Remove test event hacks * remove types * removed chain_builder + added logdb to testchain * pr comments * Update test/testchain/chain.go Co-authored-by: libotony --------- Co-authored-by: libotony * cmd/thor: update instance dir to v4 * trie: implement varint-prefix coder * deps: add github.com/qianbin/drlp * trie: implement appendHexToCompact & compactLen * trie: temporarily remove merkle proof stuff * trie: many changes * disk usage reduced by 33% (force embedding shortnode) * new encoding method for storing nodes * optimize trie hashing * versioning standalone nodes * remove extended trie * improve trie interface * simplify NodeIterator, remove unused codes * trie: optimize full-node encoding/decoding * trie: tweak shortnode encoding * muxdb: move engine pkg * trie: add Version() method for node interface * muxdb: refactor due to trie updates and: * remove leafbank stuff * simplify muxdb.Trie implementation * improve root node cache using ttl eviction * add leaf key filter * chain: a lot of changes * improve block content storage scheme * remove steady block tracking * remove tx & receipt cache * state: changes due to update of trie * lowrlp: remove this pkg * txpool: changes due to underlying pkg update * genesis: changes due to underlying pkg update * consensus: changes due to underlying pkg update * builtin: changes due to underlying pkg update * runtime: changes due to underlying pkg update * api: changes due to underlying pkg update * cmd/thor/pruner: rename pkg optimizer to pruner * cmd/thor: changes due to underlying pkg update * muxdb: abandon leaf filter * cmd/thor/pruner: use smaller period when nearly synced * muxdb: improve trie node path encoding * trie: treat short nodes as standalone nodes when skipping hash * cmd/thor: fix disablePrunerFlag not work * trie: improve refNode encoding/decoding * muxdb: improve history node key encoding * cmd/thor: adjust pruner parameters * build: fix test cases * lint: fix lint error * muxdb: fix ver encoding in node blob cache * muxdb: add test cases for cache * runtime: fix test compile error * make build and test pass after rebase * add back pruner tests * add tests for node encoding * minor typo * update named store space prefix * add more tests * fix block summary in repo * make build and test pass after rebase * add back pruner tests * remove SetBestBlockID from tests * minor improvement * pr comments * adding a comment * Metrics: Cache hit/miss (#886) * change * reverted to previous format * Add dummy cache for inmem ops (#883) * Add empty cache for inmem ops * changing name to dummyCache * Metrics: Reuse `shouldLog` so we get cache hit/miss data at the same pace (#888) * first commit * first commit * Metrics: Disk IO reads and writes (#890) * changes * removed log * sleeping any way * pr review * Pedro/maindb v4/benchmarks (#891) * Adding Benchmark tests * processing txs * Working benchmarks * lint * adding tempdir * improve cache stats log and metric * totally removed SetBestBlockID * Maindb v4 Transaction benchmark plus cache (#894) * Thor client (#818) * feat: add thorclient * refactor: remove roundTripper * refactor: change null check * clean: remove commented code * feat: add account revision and pending tx * fix: add licence headers and fix linter issue * refactor: rename package * refactor: change revision type to string * refactor: rename GetLogs and GetTransfers to FilterEvents and FilterTransfers * refactor: change FilterEvents and FilterTransactions request type to EventFilter * Adding common.EventWrapper to handle channel errors * tweak * update rawclient + update account tests * tidy up names * update tests * pr comments * adding raw tx * Tidy up method names and calls * options client * tweaks * pr comments * Update thorclient/common/common.go Co-authored-by: libotony * pr comments * Adding Subscriptions * Pr comments * adjust func orders * pr comments * changing subscribe to use the channel close vs multiple channels * adding go-doc * no error after unsubscribe * pr comments * checking status code is 2xx * fix: change FilterTransfers argument --------- Co-authored-by: otherview Co-authored-by: libotony * Add testchain package (#844) * Refactor thor node * thorchain allows insertion of blocks * remove thorNode, added testchain * clean up + comments * adding license headers * adding templating tests for thorclient * Remove test event hacks * remove types * removed chain_builder + added logdb to testchain * pr comments * Update test/testchain/chain.go Co-authored-by: libotony --------- Co-authored-by: libotony * cmd/thor: update instance dir to v4 * trie: implement varint-prefix coder * deps: add github.com/qianbin/drlp * trie: implement appendHexToCompact & compactLen * trie: temporarily remove merkle proof stuff * trie: many changes * disk usage reduced by 33% (force embedding shortnode) * new encoding method for storing nodes * optimize trie hashing * versioning standalone nodes * remove extended trie * improve trie interface * simplify NodeIterator, remove unused codes * trie: optimize full-node encoding/decoding * trie: tweak shortnode encoding * muxdb: move engine pkg * trie: add Version() method for node interface * muxdb: refactor due to trie updates and: * remove leafbank stuff * simplify muxdb.Trie implementation * improve root node cache using ttl eviction * add leaf key filter * chain: a lot of changes * improve block content storage scheme * remove steady block tracking * remove tx & receipt cache * state: changes due to update of trie * lowrlp: remove this pkg * txpool: changes due to underlying pkg update * genesis: changes due to underlying pkg update * consensus: changes due to underlying pkg update * builtin: changes due to underlying pkg update * runtime: changes due to underlying pkg update * api: changes due to underlying pkg update * cmd/thor/pruner: rename pkg optimizer to pruner * cmd/thor: changes due to underlying pkg update * muxdb: abandon leaf filter * cmd/thor/pruner: use smaller period when nearly synced * muxdb: improve trie node path encoding * trie: treat short nodes as standalone nodes when skipping hash * cmd/thor: fix disablePrunerFlag not work * trie: improve refNode encoding/decoding * muxdb: improve history node key encoding * cmd/thor: adjust pruner parameters * build: fix test cases * lint: fix lint error * muxdb: fix ver encoding in node blob cache * muxdb: add test cases for cache * runtime: fix test compile error * make build and test pass after rebase * add back pruner tests * add tests for node encoding * minor typo * update named store space prefix * add more tests * fix block summary in repo * make build and test pass after rebase * add back pruner tests * remove SetBestBlockID from tests * minor improvement * pr comments * adding a comment * Metrics: Cache hit/miss (#886) * change * reverted to previous format * Add dummy cache for inmem ops (#883) * Add empty cache for inmem ops * changing name to dummyCache * Metrics: Reuse `shouldLog` so we get cache hit/miss data at the same pace (#888) * first commit * first commit * Metrics: Disk IO reads and writes (#890) * changes * removed log * sleeping any way * pr review * Pedro/maindb v4/benchmarks (#891) * Adding Benchmark tests * processing txs * Working benchmarks * lint * adding tempdir * Adding transactions benchmark + repository cache * improve cache stats log and metric * totally removed SetBestBlockID * removed unused tests * update bench tests * getreceipts metrics + lint --------- Co-authored-by: Paolo Galli Co-authored-by: libotony Co-authored-by: qianbin Co-authored-by: Darren Kelly Co-authored-by: Miguel Angel Rojo * reduce clauses() allocations * bug: fix logs endpoints query (#900) * Thor client (#818) * feat: add thorclient * refactor: remove roundTripper * refactor: change null check * clean: remove commented code * feat: add account revision and pending tx * fix: add licence headers and fix linter issue * refactor: rename package * refactor: change revision type to string * refactor: rename GetLogs and GetTransfers to FilterEvents and FilterTransfers * refactor: change FilterEvents and FilterTransactions request type to EventFilter * Adding common.EventWrapper to handle channel errors * tweak * update rawclient + update account tests * tidy up names * update tests * pr comments * adding raw tx * Tidy up method names and calls * options client * tweaks * pr comments * Update thorclient/common/common.go Co-authored-by: libotony * pr comments * Adding Subscriptions * Pr comments * adjust func orders * pr comments * changing subscribe to use the channel close vs multiple channels * adding go-doc * no error after unsubscribe * pr comments * checking status code is 2xx * fix: change FilterTransfers argument --------- Co-authored-by: otherview Co-authored-by: libotony * Add testchain package (#844) * Refactor thor node * thorchain allows insertion of blocks * remove thorNode, added testchain * clean up + comments * adding license headers * adding templating tests for thorclient * Remove test event hacks * remove types * removed chain_builder + added logdb to testchain * pr comments * Update test/testchain/chain.go Co-authored-by: libotony --------- Co-authored-by: libotony * cmd/thor: update instance dir to v4 * trie: implement varint-prefix coder * deps: add github.com/qianbin/drlp * trie: implement appendHexToCompact & compactLen * trie: temporarily remove merkle proof stuff * trie: many changes * disk usage reduced by 33% (force embedding shortnode) * new encoding method for storing nodes * optimize trie hashing * versioning standalone nodes * remove extended trie * improve trie interface * simplify NodeIterator, remove unused codes * trie: optimize full-node encoding/decoding * trie: tweak shortnode encoding * muxdb: move engine pkg * trie: add Version() method for node interface * muxdb: refactor due to trie updates and: * remove leafbank stuff * simplify muxdb.Trie implementation * improve root node cache using ttl eviction * add leaf key filter * chain: a lot of changes * improve block content storage scheme * remove steady block tracking * remove tx & receipt cache * state: changes due to update of trie * lowrlp: remove this pkg * txpool: changes due to underlying pkg update * genesis: changes due to underlying pkg update * consensus: changes due to underlying pkg update * builtin: changes due to underlying pkg update * runtime: changes due to underlying pkg update * api: changes due to underlying pkg update * cmd/thor/pruner: rename pkg optimizer to pruner * cmd/thor: changes due to underlying pkg update * muxdb: abandon leaf filter * cmd/thor/pruner: use smaller period when nearly synced * muxdb: improve trie node path encoding * trie: treat short nodes as standalone nodes when skipping hash * cmd/thor: fix disablePrunerFlag not work * trie: improve refNode encoding/decoding * muxdb: improve history node key encoding * cmd/thor: adjust pruner parameters * build: fix test cases * lint: fix lint error * muxdb: fix ver encoding in node blob cache * muxdb: add test cases for cache * runtime: fix test compile error * make build and test pass after rebase * add back pruner tests * add tests for node encoding * minor typo * update named store space prefix * add more tests * fix block summary in repo * make build and test pass after rebase * add back pruner tests * remove SetBestBlockID from tests * minor improvement * pr comments * adding a comment * Metrics: Cache hit/miss (#886) * change * reverted to previous format * Add dummy cache for inmem ops (#883) * Add empty cache for inmem ops * changing name to dummyCache * Metrics: Reuse `shouldLog` so we get cache hit/miss data at the same pace (#888) * first commit * first commit * Metrics: Disk IO reads and writes (#890) * changes * removed log * sleeping any way * pr review * Pedro/maindb v4/benchmarks (#891) * Adding Benchmark tests * processing txs * Working benchmarks * lint * adding tempdir * improve cache stats log and metric * totally removed SetBestBlockID * fix: logs API not returning results when to=0,from=omitted * minor update * fix: PR comments + lint * improve convert range logic * chore: remove debug log --------- Co-authored-by: Paolo Galli Co-authored-by: otherview Co-authored-by: libotony Co-authored-by: qianbin Co-authored-by: Miguel Angel Rojo * chore(chain): add repo cache metrics (#910) * chore(chain): add repo cache metrics * refactor(chain): cache hit miss --------- Co-authored-by: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Co-authored-by: Pedro Gomes Co-authored-by: Paolo Galli Co-authored-by: qianbin Co-authored-by: Miguel Angel Rojo Co-authored-by: Darren Kelly * Pedro/merge/feat/db (#914) * fix(documentation): use absolute links in markdown (#889) * Add benchmark test to node block process (#892) * Add benchmark test to node block process * added file-based storage * use tempdir * update dependency go-ethereum (#895) * chore: update API metrics bucket and endpoint names (#893) * chore: update API metrics bucket and endpoint names * fix: typo & tests * fix: lint * chore: add websocket total counter * fix: txs endpoints names & ws subject * fix: unit tests * chore: standardise naming convention * chore: add websocke duration & http code * chore: add websocke duration & http code * fix: lint issues * fix: sync issues with metrics * chore: update websocket durations bucket * fix: PR comments - use sync.Once * chore: update builtin generation (#896) * chore: update builtin generation * fix: update GHA * getreceipts metrics + lint (#902) * chore: add flag to enable/disable deprecated APIs (#897) * chore: add flag to enable/disable deprecated APIs * chore: update for PR comments * chore: update for PR comments * fix: update e2e commit sha * fix: update e2e commit sha * fix: update flag name * fix: solo start flags (#906) * chore: make thorclient configurable + fix type error (#908) * chore: make thorclient configurable * fix: subscriptions block type * fix: compile errors * fix: remove test with lint error * add 'raw' query parameter to the blocks (#899) * add 'raw' query parameter to the blocks * summary -> summary.Header Co-authored-by: libotony * change variable name * make expanded and raw mutually exclusive * add unit tests * fix linting --------- Co-authored-by: libotony * Adding Health endpoint (#836) * Adding Health endpoint * pr comments + 503 if not healthy * refactored admin server and api + health endpoint tests * fix health condition * fix admin routing * added comments + changed from ChainSync to ChainBootstrapStatus * Adding healthcheck for solo mode * adding solo + tests * fix log_level handler funcs * refactor health package + add p2p count * remove solo methods * moving health service to api pkg * added defaults + api health query * pr comments * pr comments * pr comments * Update cmd/thor/main.go * Darren/admin api log toggler (#877) * Adding Health endpoint * pr comments + 503 if not healthy * refactored admin server and api + health endpoint tests * fix health condition * fix admin routing * added comments + changed from ChainSync to ChainBootstrapStatus * Adding healthcheck for solo mode * adding solo + tests * fix log_level handler funcs * feat(admin): toggle api logs via admin API * feat(admin): add license headers * refactor health package + add p2p count * remove solo methods * moving health service to api pkg * added defaults + api health query * pr comments * pr comments --------- Co-authored-by: otherview * Darren/chore/backport metrics (#909) * chore(muxdb): backport muxdb cache metrics * chore(muxdb): backport muxdb cache metrics * chore(metrics): backport disk IO * chore(metrics): fix lint * chore(chain): add repo cache metrics * fix(chain): fix cache return value * refactor(chain): cache hit miss * chore(thor): update version (#912) * chore(thor): update version * chore(openapi): version * feat(api/debug): support debug trace without blockId (#905) * api/debug: support debug with txhash Signed-off-by: jsvisa api/debug: blockId should use tx's instead Signed-off-by: jsvisa fix tests Signed-off-by: jsvisa * debug: add test Signed-off-by: jsvisa * improve parseTarget Signed-off-by: jsvisa * update doc Signed-off-by: jsvisa * fix tests Signed-off-by: jsvisa --------- Signed-off-by: jsvisa Co-authored-by: tony * version --------- Signed-off-by: jsvisa Co-authored-by: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Co-authored-by: libotony Co-authored-by: YeahNotSewerSide <47860375+YeahNotSewerSide@users.noreply.github.com> Co-authored-by: Delweng * fix: compile errors * added compaction metrics * added compaction metrics * chore(maindbv4): add a migration guide" (#944) * Update muxdb/metrics.go Co-authored-by: Pedro Gomes * Update muxdb/metrics.go Co-authored-by: Pedro Gomes * Update muxdb/metrics.go Co-authored-by: Pedro Gomes * code review comments * cancelling context when closing muxdb * added tests * license headers * license headers * cancelfunc moved after comments * added waitgroup --------- Signed-off-by: jsvisa Co-authored-by: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Co-authored-by: Paolo Galli Co-authored-by: otherview Co-authored-by: libotony Co-authored-by: Makis Christou Co-authored-by: Darren Kelly Co-authored-by: qianbin Co-authored-by: YeahNotSewerSide <47860375+YeahNotSewerSide@users.noreply.github.com> Co-authored-by: Delweng --- muxdb/metrics.go | 137 +++++++++++++++++++++++++++++++++++++++++- muxdb/metrics_test.go | 36 +++++++++++ muxdb/muxdb.go | 47 ++++++++++++++- 3 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 muxdb/metrics_test.go diff --git a/muxdb/metrics.go b/muxdb/metrics.go index 65ead18fb..298b75bed 100644 --- a/muxdb/metrics.go +++ b/muxdb/metrics.go @@ -1,4 +1,4 @@ -// Copyright (c) 2024 The VeChainThor developers +// Copyright (c) 2025 The VeChainThor developers // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or @@ -8,7 +8,140 @@ package muxdb import ( + "fmt" + "math" + "strconv" + "strings" + "text/tabwriter" + "github.com/vechain/thor/v2/metrics" ) -var metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"}) +var ( + metricCacheHitMiss = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"}) + metricCompaction = metrics.LazyLoadGaugeVec("compaction_stats_gauge", []string{"level", "type"}) +) + +// CompactionValues holds the values for a specific level. +type CompactionValues struct { + Level string + Tables int64 + SizeMB int64 + TimeSec int64 + ReadMB int64 + WriteMB int64 +} + +// Collects the compaction values from the stats table. +// The format of the stats table is: +/* +Compactions + Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 2 | 224.46577 | 3.25844 | 0.00000 | 1908.26756 + 1 | 29 | 110.98547 | 6.76062 | 2070.73768 | 2054.52797 + 2 | 295 | 1109.32673 | 3.16157 | 883.22560 | 799.85596 + 3 | 2777 | 10206.97173 | 0.33533 | 103.17983 | 91.55081 + 4 | 4100 | 15773.54834 | 6.75241 | 2032.57337 | 1851.48528 +-------+------------+---------------+---------------+---------------+--------------- + Total | 7203 | 27425.29804 | 20.26837 | 5089.71648 | 6705.68758 + +*/ +func collectCompactionValues(stats string) { + // Create a new tabwriter + var sb strings.Builder + w := tabwriter.NewWriter(&sb, 0, 0, 1, ' ', tabwriter.Debug) + + // Print the stats string using the tabwriter + fmt.Fprintln(w, stats) + w.Flush() + + // Extract and log the value from the specified level + formattedStats := sb.String() + logger.Debug(formattedStats) + values, err := extractCompactionValues(formattedStats) + if err != nil { + logger.Error("Failed to extract values for stats %s: %s", stats, err.Error()) + } else { + for _, value := range values { + metricCompaction().SetWithLabel(value.Tables, map[string]string{"level": value.Level, "type": "tables"}) + metricCompaction().SetWithLabel(value.SizeMB, map[string]string{"level": value.Level, "type": "size-mb"}) + metricCompaction().SetWithLabel(value.TimeSec, map[string]string{"level": value.Level, "type": "time-sec"}) + metricCompaction().SetWithLabel(value.ReadMB, map[string]string{"level": value.Level, "type": "read-mb"}) + metricCompaction().SetWithLabel(value.WriteMB, map[string]string{"level": value.Level, "type": "write-mb"}) + } + } +} + +func parseAndRoundFloatToInt64(str string) (int64, error) { + // Parse the string to a float64 + floatValue, err := strconv.ParseFloat(str, 64) + if err != nil { + return 0, err + } + + // Round the float64 value + roundedValue := math.Round(floatValue) + + // Convert the rounded float64 to int64 + intValue := int64(roundedValue) + + return intValue, nil +} + +func extractCompactionValues(stats string) ([]CompactionValues, error) { + lines := strings.Split(stats, "\n") + var values []CompactionValues + + if len(lines) < 6 { + return nil, fmt.Errorf("not enough lines in stats %s", stats) + } + + for _, line := range lines[2 : len(lines)-3] { + columns := strings.Fields(line) + if len(columns) >= 6 { + value, err := parseCompactionColumns(columns) + if err != nil { + return nil, err + } + values = append(values, *value) + } + } + + if len(values) == 0 { + return nil, fmt.Errorf("no valid compaction values found in stats %s", stats) + } + + return values, nil +} + +func parseCompactionColumns(columns []string) (*CompactionValues, error) { + tables, err := strconv.ParseInt(columns[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("error when parsing tables: %v", err) + } + sizeMb, err := parseAndRoundFloatToInt64(columns[4]) + if err != nil { + return nil, fmt.Errorf("error when parsing sizeMb: %v", err) + } + timeSec, err := parseAndRoundFloatToInt64(columns[6]) + if err != nil { + return nil, fmt.Errorf("error when parsing timeSec: %v", err) + } + readMb, err := parseAndRoundFloatToInt64(columns[8]) + if err != nil { + return nil, fmt.Errorf("error when parsing readMb: %v", err) + } + writeMb, err := parseAndRoundFloatToInt64(columns[10]) + if err != nil { + return nil, fmt.Errorf("error when parsing writeMb: %v", err) + } + return &CompactionValues{ + Level: columns[0], + Tables: tables, + SizeMB: sizeMb, + TimeSec: timeSec, + ReadMB: readMb, + WriteMB: writeMb, + }, nil +} diff --git a/muxdb/metrics_test.go b/muxdb/metrics_test.go new file mode 100644 index 000000000..b4fdba2f0 --- /dev/null +++ b/muxdb/metrics_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2025 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +// Package muxdb implements the storage layer for block-chain. +// It manages instance of merkle-patricia-trie, and general purpose named kv-store. +package muxdb + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMetricsCollectCompactionValues(t *testing.T) { + require.NotPanics(t, func() { collectCompactionValues("") }) + require.NotPanics(t, func() { collectCompactionValues("wrong stats") }) + + stats := ` Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 0 | 0.00000 | 0.16840 | 0.00000 | 61.67909 + 1 | 27 | 96.34199 | 1.03280 | 139.39040 | 138.68919 + 2 | 271 | 989.34527 | 0.15046 | 45.49008 | 39.92714 + 3 | 2732 | 10002.10112 | 1.11660 | 128.58780 | 119.32566 + 4 | 3544 | 13591.24199 | 3.38804 | 2059.54114 | 223.60823 +-------+------------+---------------+---------------+---------------+--------------- + Total | 6574 | 24679.03037 | 5.85630 | 2373.00942 | 583.22930 +` + values, err := extractCompactionValues(stats) + + require.Equal(t, nil, err) + require.Equal(t, "0", values[0].Level) + require.Equal(t, int64(27), values[1].Tables) + require.Equal(t, int64(989), values[2].SizeMB) +} diff --git a/muxdb/muxdb.go b/muxdb/muxdb.go index a0f00ae2c..eca46f414 100644 --- a/muxdb/muxdb.go +++ b/muxdb/muxdb.go @@ -10,6 +10,8 @@ package muxdb import ( "context" "encoding/json" + "sync" + "time" "github.com/syndtr/goleveldb/leveldb" dberrors "github.com/syndtr/goleveldb/leveldb/errors" @@ -60,6 +62,29 @@ type Options struct { type MuxDB struct { engine engine.Engine trieBackend *backend + cancelFunc context.CancelFunc + wg sync.WaitGroup +} + +// collectCompactionMetrics collects compaction metrics periodically. +func collectCompactionMetrics(ctx context.Context, ldb *leveldb.DB) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // stats is a table in a single string, so we need to parse it + stats, err := ldb.GetProperty("leveldb.stats") + if err != nil { + logger.Error("Failed to get LevelDB stats: %v", err) + } else { + collectCompactionValues(stats) + } + case <-ctx.Done(): + return + } + } } // Open opens or creates DB at the given path. @@ -103,7 +128,9 @@ func Open(path string, options *Options) (*MuxDB, error) { return nil, err } - return &MuxDB{ + ctx, cancelFunc := context.WithCancel(context.Background()) + + muxdb := &MuxDB{ engine: engine, trieBackend: &backend{ Store: engine, @@ -114,7 +141,16 @@ func Open(path string, options *Options) (*MuxDB, error) { DedupedPtnFactor: cfg.DedupedPtnFactor, CachedNodeTTL: options.TrieCachedNodeTTL, }, - }, nil + cancelFunc: cancelFunc, + } + + muxdb.wg.Add(1) + go func() { + defer muxdb.wg.Done() + collectCompactionMetrics(ctx, ldb) + }() + + return muxdb, nil } // NewMem creates a memory-backed DB. @@ -137,6 +173,13 @@ func NewMem() *MuxDB { // Close closes the DB. func (db *MuxDB) Close() error { + if db.cancelFunc != nil { + db.cancelFunc() + } + + // Wait for all goroutines to finish + db.wg.Wait() + return db.engine.Close() }